viral_seq 1.0.12 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97ecf609d8927a59e1174ea5f01794e3e9c1f598cf547f72cc53a66465817c0e
-  data.tar.gz: d9c8deeb6452075aa49a63f02e8e6a6f58ef10d8969065987ffa05651e2ad2b8
+  metadata.gz: '093a8d1d70e251b0748b7035c829eb512991437ffa78fd67387318412e54acf5'
+  data.tar.gz: 1b9d6f6b2cb2ffa8d9cc588b8df096e7ac3840c694bfb241fcf970b738899328
 SHA512:
-  metadata.gz: 2f8dc3fd2c8e5f8cb0bdd130451a186790d1b9d102b66851f24d843368926632e375515e81df3e62b60dee8400effb17e9e15fcc74eab4b97c6f33dbc5cba358
-  data.tar.gz: a53032e574e844411c8b11ed07f238cba1a795f819fb3cae3d5cca129f7bcd38312efb30a87520b35540915ff461ce03e4448a6cdade50bda1c77497b5a03aef
+  metadata.gz: 3853dbfa3f6604d907ec3d77b8c86ec8d885fedcc854c40ca6822ec72e8b2cfe9413bc188aa722a14e4e4f6c9503eca1b36d7f8e0963a5a997c9f0ca8b54fc86
+  data.tar.gz: e5b056cddcf7b87cc30e52c878879cea82d865ea7fc867535767918c30c699d58d6f426518aad02be49916c49f38d9603b0ab27ca6f3625f7a5102ae86863023

data/.gitignore

@@ -2,7 +2,6 @@
 /.yardoc
 /_yardoc/
 /coverage/
-/doc/
 /pkg/
 /spec/reports/
 /tmp/

data/Gemfile.lock

@@ -1,16 +1,27 @@
 PATH
   remote: .
   specs:
-    viral_seq (1.0.10)
-      colorize (~> 0.1)
-      muscle_bio (~> 0.4)
+    viral_seq (1.1.1)
+      colorize (>= 0.1)
+      combine_pdf (>= 1.0.0)
+      muscle_bio (>= 0.4)
+      prawn (>= 2.3.0)
+      prawn-table (>= 0.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     colorize (0.8.1)
+    combine_pdf (1.0.21)
+      ruby-rc4 (>= 0.1.5)
     diff-lcs (1.3)
     muscle_bio (0.4.0)
+    pdf-core (0.9.0)
+    prawn (2.4.0)
+      pdf-core (~> 0.9.0)
+      ttfunk (~> 1.7)
+    prawn-table (0.2.2)
+      prawn (>= 1.3.0, < 3.0.0)
     rake (13.0.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
@@ -25,6 +36,8 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
+    ruby-rc4 (0.1.5)
+    ttfunk (1.7.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -1,8 +1,24 @@
 # ViralSeq
 
+[![Gem Version](https://badge.fury.io/rb/viral_seq.svg)](https://rubygems.org/gems/viral_seq)
+![GitHub](https://img.shields.io/github/license/viralseq/viral_seq)
+![Gem](https://img.shields.io/gem/dt/viral_seq?color=%23E9967A)
+![GitHub last commit](https://img.shields.io/github/last-commit/viralseq/viral_seq?color=%2300BFFF)
+[![Join the chat at https://gitter.im/viral_seq/community](https://badges.gitter.im/viral_seq/community.svg)](https://gitter.im/viral_seq/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
 A Ruby Gem containing bioinformatics tools for processing viral NGS data.
 
-Specifically for Primer-ID sequencing and HIV drug resistance analysis.
+Specifically for Primer ID sequencing and HIV drug resistance analysis.
+
+## Illustration for the Primer ID Sequencing
+
+
+![Primer ID Sequencing](./docs/assets/img/cover.jpg)
+
+### Reference readings on the Primer ID sequencing
+[Explantion of Primer ID sequencing](https://doi.org/10.21769/BioProtoc.3938)  
+[Primer ID MiSeq protocol](https://doi.org/10.1128/JVI.00522-15)  
+[Application of Primer ID sequencing in COVID-19 research](https://doi.org/10.1126/scitranslmed.abb5883)
 
 ## Install
 
@@ -14,20 +30,55 @@ Specifically for Primer-ID sequencing and HIV drug resistance analysis.
 
 ### Excutables
 
-Use executable `locator` to get the coordinates of the sequences on HIV/SIV reference genome from a FASTA file through a terminal
+### `tcs`  
+Use executable `tcs` pipeline to process **Primer ID MiSeq sequencing** data.
 
+Example commands:
 ```bash
-    $ locator -i sequence.fasta -o sequence.fasta.csv
+    $ tcs -p params.json # run TCS pipeline with params.json
+    $ tcs -p params.json -i DIRECTORY
+    # run TCS pipeline with params.json and DIRECTORY
+    # if DIRECTORY is not defined in params.json
+    $ tcs -dr -i DIRECTORY
+    # run tcs-dr (MPID HIV drug resistance sequencing) pipeline
+    # DIRECTORY needs to be given.
+    $ tcs -j # CLI to generate params.json
+    $ tcs -h # print out the help
 ```
 
-Use executable `tcs` pipeline to process Primer ID MiSeq sequencing data.
+[sample params.json for the tcs-dr pipeline](./docs/dr.json)
+
+---
+### `tcs_log`
+
+Use `tcs_log` script to pool run logs and TCS fasta files after one batch of `tcs` jobs.
 
+
+Example file structure:  
+```
+batch_tcs_jobs/  
+      ├── lib1  
+      ├── lib2  
+      ├── lib3  
+      ├── lib4  
+      ├── ...  
+```
+
+Example command:
 ```bash
-    $ tcs -p params.json # run TCS pipeline with params.json
-    $ tcs -j # CLI to generate params.json
-    $ tcs -h # print out the help
+    $ tcs_log batch_tcs_jobs
 ```
 
+---
+
+### `locator`  
+Use executable `locator` to get the coordinates of the sequences on HIV/SIV reference genome from a FASTA file through a terminal
+
+```bash
+    $ locator -i sequence.fasta -o sequence.fasta.csv
+```
+---
+
 ## Some Examples
 
 Load all ViralSeq classes by requiring 'viral_seq.rb' in your Ruby scripts.
@@ -80,21 +131,53 @@ qc_seqhash.sdrm_hiv_pr(cut_off)
 ```
 ## Known issues
 
-  1. have a conflict with rails.
-  2. Update on 03032021. Still have conflict. But in rails gem file, can just use `requires: false` globally and only require "viral_seq" when the module is needed in controller.
+  1. ~~have a conflict with rails.~~
+  2. ~~Update on 03032021. Still have conflict. But in rails gem file, can just use `requires: false` globally and only require "viral_seq" when the module is needed in controller.~~
+  3. The conflict seems to be resovled. It was from a combination of using `!` as a function for factorial and the gem name `viral_seq`. @_@
 
 ## Updates
 
-### Version 1.1.2-03032021
+### Version 1.1.2-04262021
+
+  1. Added function `ViralSeq::DRMs.sdrm_json` to export SDRM as json object.
+  2. Added a random string to the temp file names for `muscle_bio` to avoid issues when running scripts in parallel.
+  3. Added `--keep-original` flag to the `tcs` pipeline. 
+
+### Version 1.1.1-04012021
+
+  1. Added warning when paired_raw_sequence less than 0.1% of total_raw_sequence.
+  2. Added option `-i WORKING_DIRECTORY` to the `tcs` script.
+  If the `params.json` file does not contain the path to the working directory, it will append path to the run params.
+  3. Added option `-dr` to the `tcs` script.
+
+### Version 1.1.0-03252021
+
+  1. Optimized the algorithm of end-join.
+  2. Fixed a bug in the `tcs` pipeline that sometimes combined tcs files are not saved.
+  3. Added `tcs_log` command to pool run logs and tcs files from one batch of tcs jobs.
+  4. Added the preset of MPID-HIVDR params file [***dr.json***](./docs/dr.json) in /docs.
+  5. Add `platform_format` option in the json generator of the `tcs` Pipeline.
+  Users can choose from 3 MiSeq platforms for processing their sequencing data.
+  MiSeq 300x7x300 is the default option.
+
+### Version 1.0.14-03052021
+
+  1. Add a function `ViralSeq::TcsCore.validate_file_name` to check MiSeq paired-end file names.
+
+### Version 1.0.13-03032021
+
+  1. Fixed the conflict with rails.
+
+### Version 1.0.12-03032021
 
   1. Fixed an issue that may cause conflicts with ActiveRecord.
 
-### Version 1.1.1-03022021
+### Version 1.0.11-03022021
 
   1. Fixed an issue when calculating Poisson cutoff for minority mutations `ViralSeq::SeqHash.pm`.
   2. fixed an issue loading class 'OptionParser'in some ruby environments.
 
-### Version 1.1.0-11112020:
+### Version 1.0.10-11112020:
 
   1. Modularize TCS pipeline. Move key functions into /viral_seq/tcs_core.rb
   2. `tcs_json_generator` is removed. This CLI is delivered within the `tcs` pipeline, by running `tcs -j`. The scripts are included in the /viral_seq/tcs_json.rb

data/bin/tcs

@@ -23,7 +23,7 @@
 # THE SOFTWARE.
 
 # Use JSON file as the run param
-# run tcs_json_generator.rb to generate param json file.
+# run `tcs -j` to generate param json file.
 
 require 'viral_seq'
 require 'json'
@@ -46,11 +46,23 @@ OptionParser.new do |opts|
     options[:params_json] = p
   end
 
+  opts.on("-i", "--input PATH_TO_WORKING_DIRECTORY", "Path to the working directory") do |p|
+    options[:input] = p
+  end
+
+  opts.on("-dr", "--dr_pipeline", "HIV drug resistance MPID pipeline") do |p|
+    options[:dr] = true
+  end
+
   opts.on("-h", "--help", "Prints this help") do
     puts opts
     exit
   end
 
+  opts.on("--keep-original", "keep raw sequence files") do
+    options[:keep] = true
+  end
+
   opts.on("-v", "--version", "Version info") do
     puts "tcs version: " + ViralSeq::TCS_VERSION.red.bold
     puts "viral_seq version: " + ViralSeq::VERSION.red.bold
@@ -64,15 +76,21 @@ end.parse!
 
 if options[:json_generator]
   params = ViralSeq::TcsJson.generate
+elsif options[:dr]
+  params = ViralSeq::TcsDr::PARAMS
 elsif (options[:params_json] && File.exist?(options[:params_json]))
   params = JSON.parse(File.read(options[:params_json]), symbolize_names: true)
 else
   abort "No params JSON file found. Script terminated.".red
 end
 
-indir = params[:raw_sequence_dir]
+if options[:input]
+  indir = options[:input]
+else
+  indir = params[:raw_sequence_dir]
+end
 
-unless File.exist?(indir)
+unless indir and File.exist?(indir)
   abort "No input sequence directory found. Script terminated.".red.bold
 end
 
@@ -115,6 +133,12 @@ else
   error_rate = 0.02
 end
 
+if params[:platform_format]
+  $platform_sequencing_length = params[:platform_format]
+else
+  $platform_sequencing_length = 300
+end
+
 primers = params[:primer_pairs]
 if primers.empty?
   ViralSeq::TcsCore.log_and_abort log, "No primer information. Script terminated."
@@ -123,6 +147,7 @@ end
 
 primers.each do |primer|
   summary_json = {}
+  summary_json[:warnings] = []
   summary_json[:tcs_version] = ViralSeq::TCS_VERSION
   summary_json[:viralseq_version] = ViralSeq::VERSION
   summary_json[:runtime] = Time.now.to_s
@@ -134,6 +159,7 @@ primers.each do |primer|
   forward_primer = primer[:forward]
 
   export_raw = primer[:export_raw]
+  limit_raw = primer[:limit_raw]
 
   unless cdna_primer
     log.puts Time.now.to_s + "\t" + region + " does not have cDNA primer sequence. #{region} skipped."
@@ -175,6 +201,10 @@ primers.each do |primer|
   paired_seq_number = common_keys.size
   log.puts Time.now.to_s + "\t" +  "Paired raw sequences are : #{paired_seq_number.to_s}"
   summary_json[:paired_raw_sequence] = paired_seq_number
+  if paired_seq_number < raw_sequence_number * 0.001
+    summary_json[:warnings] <<
+      "WARNING: Filtered raw sequneces less than 0.1% of the total raw sequences. Possible contamination."
+  end
 
   common_keys.each do |seqtag|
     r1_seq = r1_passed_seq[seqtag]
@@ -236,7 +266,13 @@ primers.each do |primer|
     raw_r1_f = File.open(outfile_raw_r1, 'w')
     raw_r2_f = File.open(outfile_raw_r2, 'w')
 
-    bio_r1.keys.each do |k|
+    if limit_raw
+      raw_keys = bio_r1.keys.sample(limit_raw.to_i)
+    else
+      raw_keys = bio_r1.keys
+    end
+
+    raw_keys.each do |k|
       raw_r1_f.puts k + "_r1"
       raw_r2_f.puts k + "_r2"
       raw_r1_f.puts bio_r1[k]
@@ -273,7 +309,6 @@ primers.each do |primer|
       r1_sub_seq << bio_r1[seq_name]
       r2_sub_seq << bio_r2[seq_name]
     end
-
     #consensus name including the Primer ID and number of raw sequences of that Primer ID, library name and setname.
     consensus_name = ">" + primer_id + "_" + seq_with_same_primer_id.size.to_s + "_" + libname + "_" + region
     r1_consensus = ViralSeq::SeqHash.array(r1_sub_seq).consensus(majority_cut_off)
@@ -364,6 +399,7 @@ primers.each do |primer|
     shp = ViralSeq::SeqHashPair.fa(out_dir_consensus)
     joined_sh = end_join(out_dir_consensus, primer[:end_join_option], primer[:overlap])
     log.puts Time.now.to_s + "\t" + "Paired TCS number: " + joined_sh.size.to_s
+
     summary_json[:combined_tcs] = joined_sh.size
 
     if export_raw
@@ -433,12 +469,15 @@ primers.each do |primer|
       trim_end = primer[:trim_ref_end]
       trim_ref = primer[:trim_ref].to_sym
       joined_sh = joined_sh.trim(trim_start, trim_end, trim_ref)
-      joined_sh.write_nt_fa(File.join(out_dir_consensus, "combined.fasta"))
       if export_raw
         joined_sh_raw = joined_sh_raw.trim(trim_start, trim_end, trim_ref)
-        joined_sh_raw.write_nt_fa(File.join(out_dir_raw, "combined.raw.fasta"))
       end
     end
+
+    joined_sh.write_nt_fa(File.join(out_dir_consensus, "combined.fasta"))
+    if export_raw
+      joined_sh_raw.write_nt_fa(File.join(out_dir_raw, "combined.raw.fasta"))
+    end
   end
 
   File.open(outfile_log, "w") do |f|
@@ -446,9 +485,11 @@ primers.each do |primer|
   end
 end
 
-log.puts Time.now.to_s + "\t" + "Removing raw sequence files..."
-File.unlink(r1_f)
-File.unlink(r2_f)
+unless options[:keep]
+  log.puts Time.now.to_s + "\t" + "Removing raw sequence files..."
+  File.unlink(r1_f)
+  File.unlink(r2_f)
+end
 log.puts Time.now.to_s + "\t" + "TCS pipeline successfuly exercuted."
 log.close
 puts "DONE!"

data/bin/tcs_log

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+
+# pool run logs from one batch of tcs jobs
+# file structure:
+#   batch_tcs_jobs/
+#   ├── lib1
+#   ├── lib2
+#   ├── lib3
+#   ├── lib4
+#   ├── ...
+#
+# command example:
+#   $ tcs_log batch_tcs_jobs
+
+require 'viral_seq'
+require 'pathname'
+require 'json'
+require 'fileutils'
+
+indir = ARGV[0].chomp
+indir_basename = File.basename(indir)
+indir_dirname = File.dirname(indir)
+
+tcs_dir = File.join(indir_dirname, (indir_basename + "_tcs"))
+Dir.mkdir(tcs_dir) unless File.directory?(tcs_dir)
+
+libs = []
+Dir.chdir(indir) {libs = Dir.glob("*")}
+
+outdir2 = File.join(tcs_dir, "combined_TCS_per_lib")
+outdir3 = File.join(tcs_dir, "TCS_per_region")
+outdir4 = File.join(tcs_dir, "combined_TCS_per_region")
+
+Dir.mkdir(outdir2) unless File.directory?(outdir2)
+Dir.mkdir(outdir3) unless File.directory?(outdir3)
+Dir.mkdir(outdir4) unless File.directory?(outdir4)
+
+log_file = File.join(tcs_dir,"log.csv")
+log = File.open(log_file,'w')
+
+header = %w{
+  lib_name
+  Region
+  Raw_Sequences_per_barcode
+  R1_Raw
+  R2_Raw
+  Paired_Raw
+  Cutoff
+  PID_Length
+  Consensus1
+  Consensus2
+  Distinct_to_Raw
+  Resampling_index
+  Combined_TCS
+  Combined_TCS_after_QC
+  WARNINGS
+}
+
+log.puts header.join(',')
+libs.each do |lib|
+  Dir.mkdir(File.join(outdir2, lib)) unless File.directory?(File.join(outdir2, lib))
+  fasta_files = []
+  json_files = []
+  Dir.chdir(File.join(indir, lib)) do
+     fasta_files = Dir.glob("**/*.fasta")
+     json_files = Dir.glob("**/log.json")
+  end
+  fasta_files.each do |f|
+    path_array = Pathname(f).each_filename.to_a
+    region = path_array[0]
+    if path_array[-1] == "combined.fasta"
+      FileUtils.cp(File.join(indir, lib, f), File.join(outdir2, lib, (lib + "_" + region)))
+      Dir.mkdir(File.join(outdir4,region)) unless File.directory?(File.join(outdir4,region))
+      FileUtils.cp(File.join(indir, lib, f), File.join(outdir4, region, (lib + "_" + region)))
+    else
+      Dir.mkdir(File.join(outdir3,region)) unless File.directory?(File.join(outdir3,region))
+      Dir.mkdir(File.join(outdir3,region, lib)) unless File.directory?(File.join(outdir3,region, lib))
+      FileUtils.cp(File.join(indir, lib, f), File.join(outdir3, region, lib, (lib + "_" + region + "_" + path_array[-1])))
+    end
+  end
+
+  json_files.each do |f|
+    json_log = JSON.parse(File.read(File.join(indir, lib, f)), symbolize_names: true)
+    log.print [lib,
+               json_log[:primer_set_name],
+               json_log[:total_raw_sequence],
+               json_log[:r1_filtered_raw],
+               json_log[:r2_filtered_raw],
+               json_log[:paired_raw_sequence],
+               json_log[:consensus_cutoff],
+               json_log[:length_of_pid],
+               json_log[:total_tcs_with_ambiguities],
+               json_log[:total_tcs],
+               json_log[:distinct_to_raw],
+               json_log[:resampling_param],
+               json_log[:combined_tcs],
+               json_log[:combined_tcs_after_qc],
+               json_log[:warnings],
+             ].join(',') + "\n"
+  end
+end
+log.close