miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Pipelines/idba.pbs/run.pbs

@@ -0,0 +1,56 @@
+#!/bin/bash
+#PBS -l nodes=1:ppn=10
+#PBS -k eo
+
+module load idba/1.1.1
+
+b=$SAMPLE
+shared=/nv/gpfs-gateway-pace1/project/bio-konstantinidis/shared3
+enve=$shared/apps/enveomics/Scripts
+THR=10
+
+#---------------------------------------------------------
+
+echo "==[ 05.assembly: $(date) ]"
+cd $FOLDER/05.assembly
+
+CMD=""
+case "$TYPE" in
+*g)
+   CMD="idba_ud" ;;
+t)
+   CMD="idba_tran" ;;
+*)
+   echo "Unsupported data type: $TYPE" >&2
+   exit 1
+   ;;
+esac
+CMD="$CMD --pre_correction -r $FA -o $SAMPLE --num_threads $THR"
+[[ -n "$FA_RL2" ]] && CMD="$CMD --read_level_2 $FA_RL2"
+[[ -n "$FA_RL3" ]] && CMD="$CMD --read_level_3 $FA_RL3"
+[[ -n "$FA_RL4" ]] && CMD="$CMD --read_level_4 $FA_RL4"
+[[ -n "$FA_RL5" ]] && CMD="$CMD --read_level_5 $FA_RL5"
+
+time $CMD
+
+rm $SAMPLE/kmer
+rm $SAMPLE/graph-*.fa
+rm $SAMPLE/align-*
+rm $SAMPLE/local-contig-*.fa
+rm $SAMPLE/contig-*.fa
+
+if [[ -s $SAMPLE/scaffold.fa ]] ; then
+   ln -s $SAMPLE/scaffold.fa $SAMPLE.AllContigs.fna
+else
+   ln -s $SAMPLE/contig.fa $SAMPLE.AllContigs.fna
+fi
+time $enve/FastA.length.pl $SAMPLE.AllContigs.fna | awk '$2>=500{print $1}' \
+   > $SAMPLE.LargeContigs.ids
+time $enve/FastA.filter.pl $SAMPLE.LargeContigs.ids $SAMPLE.AllContigs.fna \
+   > $SAMPLE.LargeContigs.fna
+rm $SAMPLE.LargeContigs.ids
+
+#---------------------------------------------------------
+
+echo "Done: $(date)."
+

data/utils/enveomics/Pipelines/trim.pbs/README.md

@@ -0,0 +1,54 @@
+@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
+
+@update: Oct-30-2014
+
+@license: artistic 2.0
+
+@status: auto
+
+@pbs: yes
+
+# IMPORTANT
+
+This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
+are free to use it in other platforms with adequate adjustments.
+
+# PURPOSE
+
+Performs various trimming and quality-control analyses over raw reads.
+
+# HELP
+
+1. Files preparation:
+
+   1.1. Obtain the enveomics package in the cluster. You can use:
+      `git clone https://github.com/lmrodriguezr/enveomics.git`
+   
+   1.2. Prepare the raw reads in FastQ format. Files must be raw, not zipped or packaged.
+      Filenames must conform the format: <name>.<sis>.fastq, where <name> is the name
+      of the sample, and <sis> is 1 or 2 indicating which sister read the file contains.
+      Use only '1' as <sis> if you have single reads.
+   
+   1.3. Gather all the FastQ files into the same folder.
+
+2. Pipeline execution:
+   
+   2.1. Simply execute `./RUNME.bash <dir>`, where <dir> is the folder containing
+      the FastQ files.
+
+3. What to expect:
+
+   By the end of the run, you should find the following folders:
+   
+   3.1. *01.raw_reads*: Gzip'ed raw FastQ files.
+   
+   3.2. *02.trimmed_reads*: Trimmed and clipped reads. For each sample, there should be
+      nine files for paired-end, and two for single-reads.
+
+   3.3. *03.read_quality*: Quality reports. For each sample, there should be two directories,
+      one with SolexaQA++ information, another with FastQC information.
+
+   3.4. *04.trimmed_fasta*: Trimmed and clipped in FastA format (and gzip'ed, in the case of
+      individual files for paired-end).
+
+

data/utils/enveomics/Pipelines/trim.pbs/RUNME.bash

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+if [[ "$1" == "" || "$1" == "-h" ]] ; then
+   echo "
+   Usage: ./RUNME.bash folder [clipper [max_jobs]]
+
+   folder	Path to the folder containing the raw reads. The raw reads must be in FastQ format,
+   		and filenames must follow the format: <name>.<sis>.fastq, where <name> is the name
+		of the sample, and <sis> is 1 or 2 indicating which sister read the file contains.
+		Use only '1' as <sis> if you have single reads.
+   clipper	(optional) One of: trimmomatic, scythe, or none. By default: scythe.
+   max_jobs	(optional) Maximum number of jobs to run in parallel. This number can be increased,
+   		but bear in mind that this process is highly I/O-intensive, and likely to crash or
+		significantly slow down the hard drive if many jobs are running simultaneously. By
+		default: 5.
+   " >&2 ;
+   exit 1 ;
+fi ;
+CLIPPER=$2
+if [[ "$CLIPPER" == "" ]] ; then
+   CLIPPER="scythe"
+fi ;
+if [[ "$3" == "" ]] ; then
+   MAX=5 ;
+else
+   let MAX=$3+0 ;
+fi ;
+
+dir=$(readlink -f $1) ;
+pac=$(dirname $(readlink -f $0)) ;
+cwd=$(pwd) ;
+
+cd $dir ;
+for i in 01.raw_reads 02.trimmed_reads 03.read_quality 04.trimmed_fasta zz.info ; do
+   if [[ ! -d $i ]] ; then mkdir $i ; fi ;
+done ;
+
+k=0 ;
+for i in $dir/*.1.fastq ; do
+   EXTRA="" ;
+   EXTRA_MSG="" ;
+   if [[ $k -ge $MAX ]] ; then
+      let prek=$k-$MAX ;
+      EXTRA="-W depend=afterany:${jids[$prek]}" ;
+      EXTRA_MSG=" (waiting for ${jids[$prek]})"
+   fi ;
+   b=$(basename $i .1.fastq) ;
+   mv $b.[12].fastq 01.raw_reads/ ;
+   # Predict time (in hours)
+   SIZE_M=$(($(ls -pl 01.raw_reads/$b.1.fastq | awk '{print $5}')/1000000)) ;
+   let TIME_H=$SIZE_M*5/1000 ;
+   [[ -e 01.raw_reads/$b.2.fastq ]] || let TIME_H=$TIME_H/2 ;
+   let RAM_G=$SIZE_M*8/1000 ;
+   [[ $RAM_G -lt 10 ]] && RAM_G=10 ;
+   
+   # Find the right queue
+   if [[ $TIME_H -lt 12 ]] ; then
+      QUEUE="-q iw-shared-6 -l walltime=12:00:00" ;
+   elif [[ $TIME_H -lt 120 ]] ; then
+      QUEUE="-q microcluster -l walltime=120:00:00" ;
+   else
+      QUEUE="-q microcluster -l walltime=2000:00:00" ;
+   fi ;
+   # Launch job
+   jids[$k]=$(qsub -v "SAMPLE=$b,FOLDER=$dir,CLIPPER=$CLIPPER" -N "Trim-$b" -l "mem=${RAM_G}g" $QUEUE $EXTRA $pac/run.pbs | grep .) ;
+   echo "$b: ${jids[$k]}$EXTRA_MSG" ;
+   let k=$k+1 ;
+done ;
+
+

data/utils/enveomics/Pipelines/trim.pbs/run.pbs

@@ -0,0 +1,130 @@
+#!/bin/bash
+#PBS -l mem=10g
+#PBS -l nodes=1:ppn=1
+#PBS -k eo
+
+module load fastqc/0.11.2
+module load scythe/0.993
+
+shared=/gpfs/pace1/project/bio-konstantinidis/shared3
+b=$SAMPLE ;
+sqa=$shared/bin/SolexaQA++
+scythe=scythe
+enve=$shared/apps/enveomics/Scripts
+trim=$shared/apps/Trimmomatic-0.32/trimmomatic-0.32.jar
+SEadapters=$shared/apps/Trimmomatic-0.32/adapters/ALL-SE_PE.fa
+PEadapters=$shared/apps/Trimmomatic-0.32/adapters/ALL-PE.fa
+
+#---------------------------------------------------------
+
+echo "==[ 02.trimmed_reads: $(date) ]" ;
+cd $FOLDER/02.trimmed_reads ;
+
+time $enve/FastQ.tag.rb -i ../01.raw_reads/$b.1.fastq -p "$b-" -s "/1" -o $b.1.fastq ;
+[[ -e ../01.raw_reads/$b.2.fastq ]] && time $enve/FastQ.tag.rb -i ../01.raw_reads/$b.2.fastq -p "$b-" -s "/2" -o $b.2.fastq ;
+
+RAW_READS=$(cat $b.1.fastq | paste - - - - | wc -l | sed -e 's/ *//') ;
+RAW_LENGTH=$(head -n 40000 $b.1.fastq | paste - - - - | awk 'BEGIN{FS="\\t"}{SUM+=length($2)}END{print SUM/NR}') ;
+
+time $sqa dynamictrim $b.[12].fastq -h 20 -d . ;
+time $sqa lengthsort $b.[12].fastq.trimmed -l 50 -d . ;
+
+if [[ "$CLIPPER" == "trimmomatic" ]] ; then
+   if [[ -e $b.2.fastq.trimmed.paired ]] ; then
+      time java -jar $trim PE -threads 1 \
+	 $b.1.fastq.trimmed.paired \
+	 $b.2.fastq.trimmed.paired \
+	 $b.1.clipped.fastq $b.1.clipped.single.fastq \
+	 $b.2.clipped.fastq $b.2.clipped.single.fastq \
+	 ILLUMINACLIP:$PEadapters:2:30:10 MINLEN:50
+   else
+      time java -jar $trim SE -threads 1 \
+	 $b.1.fastq.trimmed.single $b.1.clipped.fastq \
+	 ILLUMINACLIP:$SEadapters:2:30:10 MINLEN:50
+   fi ;
+elif [[ "$CLIPPER" == "scythe" ]]; then
+   if [[ -e $b.2.fastq.trimmed.paired ]] ; then
+      $scythe -a $PEadapters $b.1.fastq.trimmed.paired > $b.1.clipped.all.fastq ;
+      $scythe -a $PEadapters $b.2.fastq.trimmed.paired > $b.2.clipped.all.fastq ;
+      time $sqa lengthsort $b.[12].clipped.all.fastq -l 50 -d . ;
+      rm $b.[12].clipped.all.fastq ;
+      [[ -e $b.1.clipped.all.fastq.single ]] && mv $b.1.clipped.all.fastq.single $b.1.clipped.single.fastq ;
+      [[ -e $b.2.clipped.all.fastq.single ]] && mv $b.2.clipped.all.fastq.single $b.2.clipped.single.fastq ;
+      mv $b.1.clipped.all.fastq.paired $b.1.clipped.fastq ;
+      mv $b.2.clipped.all.fastq.paired $b.2.clipped.fastq ;
+      rm $b.1.clipped.all.fastq.summary.txt $b.1.clipped.all.fastq.summary.txt.pdf &>/dev/null ;
+   else
+      $scythe -a $PEadapters $b.1.fastq.trimmed.single > $b.1.clipped.all.fastq ;
+      time $sqa lengthsort $b.1.clipped.all.fastq -l 50 -d . ;
+      rm $b.1.clipped.all.fastq ;
+      mv $b.1.clipped.all.fastq.single $b.1.clipped.fastq ;
+   fi ;
+   rm $b.[12].*.discard &>/dev/null ;
+else
+   if [[ -e $b.2.fastq.trimmed.paired ]] ; then
+      ln -s $b.1.fastq.trimmed.paired $b.1.clipped.fastq ;
+      ln -s $b.2.fastq.trimmed.paired $b.2.clipped.fastq ;
+   else
+      ln -s $b.1.fastq.trimmed.single $b.1.clipped.fastq ;
+   fi ;
+fi ;
+
+TRIMMED_READS=$(cat $b.1.clipped.fastq | paste - - - - | wc -l | sed -e 's/ *//') ;
+TRIMMED_LENGTH=$(head -n 40000 $b.1.clipped.fastq | paste - - - - | awk 'BEGIN{FS="\\t"}{SUM+=length($2)}END{print SUM/NR}') ;
+
+#---------------------------------------------------------
+
+echo "==[ 03.read_quality: $(date) ]" ;
+cd $FOLDER/03.read_quality ;
+if [ ! -d $b.fastqc ] ; then mkdir $b.fastqc ; fi ;
+perl $(which fastqc) ../02.trimmed_reads/$b.[12].clipped.fastq -o $b.fastqc ;
+
+if [ ! -d $b ] ; then mkdir $b ; fi ;
+time $sqa analysis ../01.raw_reads/$b.[12].fastq -h 20 -d $b -v -m ;
+rm $b/*.segments ;
+mv ../02.trimmed_reads/$b.[12].fastq_trimmed.segments* $b/
+mv ../02.trimmed_reads/$b.[12].fastq.trimmed.summary.txt* $b/
+
+
+cd $FOLDER/02.trimmed_reads ;
+rm $b.[12].fastq.trimmed.discard ;
+rm $b.[12].fastq.trimmed ;
+rm $b.[12].fastq ;
+
+#---------------------------------------------------------
+
+echo "==[ 04.trimmed_fasta: $(date) ]" ;
+cd $FOLDER/04.trimmed_fasta ;
+cat ../02.trimmed_reads/$b.1.clipped.fastq | paste - - - - | awk 'BEGIN{FS="\\t"}{print ">"substr($1,2)"\\n"$2}' > $b.1.fasta ;
+if [[ -e ../02.trimmed_reads/$b.2.clipped.fastq ]] ; then
+   cat ../02.trimmed_reads/$b.2.clipped.fastq | paste - - - - | awk 'BEGIN{FS="\\t"}{print ">"substr($1,2)"\\n"$2}' > $b.2.fasta ;
+   time $enve/FastA.interpose.pl $b.CoupledReads.fa $b.[12].fasta ;
+   time gzip $b.2.fasta ;
+   time gzip $b.1.fasta ;
+else
+   mv $b.1.fasta $b.SingleReads.fa ;
+fi ;
+
+#---------------------------------------------------------
+
+echo "==[  zz.info: $(date) ]" ;
+cd $FOLDER/zz.info ;
+echo "
+RAW_LENGTH:      $RAW_LENGTH
+RAW_READS:       $RAW_READS
+TRIMMED_LENGTH:  $TRIMMED_LENGTH
+TRIMMED_READS:   $TRIMMED_READS
+" > $b.summary.txt ;
+
+#---------------------------------------------------------
+
+echo "==[ 01.raw_reads: $(date) ]"
+cd $FOLDER/01.raw_reads ;
+for i in $b.[12].fastq ; do
+   time gzip $i ;
+done ;
+
+#---------------------------------------------------------
+
+echo "Done: $(date)." ;
+

data/utils/enveomics/README.md

@@ -0,0 +1,40 @@
+# Enveomics Collection
+
+Scripts and reference libraries at [Kostas lab](http://enve-omics.gatech.edu).
+
+## Prerequisites
+
+The enveomics collection as a whole has very modest requirements, essentially a
+*nix system with `bash`, `perl`, `ruby`, and `R`. Some scripts may require
+additional libraries, or even external Software, but you'll be forewarned about
+these requirements in the documentation accompanying each script. If you prefer,
+you can also use the Graphical User Interface (GUI), that comes with additional
+tests to let you know if your system is ready to use any given script.
+
+## Graphical User Interface (GUI)
+
+The enveomics collection now has a graphical user interface! To learn more,
+please visit [enveomics-gui](https://github.com/lmrodriguezr/enveomics-gui).
+
+## License
+
+The files in this repository are licensed under the terms of the
+Artistic License 2.0, except when otherwise noted.
+
+You can find a copy of the license in [LICENSE.txt](LICENSE.txt) or at
+http://www.perlfoundation.org/artistic_license_2_0.
+
+## Documentation
+
+Most scripts in this repository are self-documented.  However,
+more extensive documentation (and some discussion) can be found at the
+[documentation website](http://enve-omics.ce.gatech.edu/enveomics/docs).
+
+## Citation
+
+If you use any of the utilitites in the Enveomics Collection in your research
+please cite:
+
+> Rodriguez-R LM & Konstantinidis KT (2016). The enveomics collection: a toolbox
+> for specialized analyses of microbial genomes and metagenomes.
+> [PeerJ Preprints 4:e1900v1](https://peerj.com/preprints/1900/).

data/utils/enveomics/Scripts/AAsubs.log2ratio.rb

@@ -0,0 +1,171 @@
+#!/usr/bin/env ruby
+#
+# @author  Luis M. Rodriguez-R
+# @update  Dec-21-2015
+# @license artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
+require "enveomics_rb/enveomics"
+
+o = {permutations: 1000, bootstraps: 1000, overwrite: false}
+OptionParser.new do |opt|
+   opt.banner = "
+   Estimates the log2-ratio of different amino acids in homologous sites using
+   an AAsubs file (see BlastPairwise.AAsubs.pl). It provides the point
+   estimation (.obs file), the bootstrap of the estimation (.boot file) and the
+   null model based on label-permutation (.null file).
+
+   Usage: #{$0} [options]".gsub(/^ +/,"")
+   opt.separator ""
+   opt.separator "Mandatory"
+   opt.on("-i", "--input FILE",
+      "Input file in AAsubs format (see BlastPairwise.AAsubs.pl)."
+      ){ |v| o[:file] = v}
+   opt.separator ""
+   opt.separator "Output files"
+   opt.on("-O", "--obs-file FILE",
+      "Output file with the log2-ratios per amino acid.",
+      "By default, '--input value'.obs."
+      ){ |v| o[:obs] = v }
+   opt.on("-B", "--bootstrap-file FILE",
+      "Output file with the bootstrap results of log2-ratios per amino acid.",
+      "By default, '--input value'.boot."
+      ){ |v| o[:boot] = v }
+   opt.on("-N", "--null-file FILE",
+      "Output file with the permutation results of log2-ratios per amino acid.",
+      "By default, '--input value'.null."
+      ){ |v| o[:null] = v }
+   opt.on("--overwrite",
+      "Overwrite existing files. By default, skip steps if the files already" +
+      " exist."){ |v| o[:overwrite] = v }
+   opt.separator ""
+   opt.separator "Parameters"
+   opt.on("-b", "--bootstraps INT",
+      "Number of bootstraps to run. By default: #{o[:bootstraps]}."
+      ){ |v| o[:bootstraps] = v.to_i }
+   opt.on("-p", "--permutations INT",
+      "Number of permutations to run. By default: #{o[:permutations]}."
+      ){ |v| o[:permutations] = v.to_i }
+   opt.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opt.on("-h", "--help", "Display this screen.") do
+      puts opt
+      exit
+   end
+   opt.separator ""
+end.parse!
+
+# Initialize
+abort "--input is mandatory" if o[:file].nil?
+ALPHABET = %w(A C D E F G H I K L M N P Q R S T V W Y X)
+o[:obs] ||= "#{o[:file]}.obs"
+o[:boot] ||= "#{o[:file]}.boot"
+o[:null] ||= "#{o[:file]}.null"
+
+# Functions
+def dist_summary(a,b)
+   ALPHABET.map do |i|
+      Math.log(a[i].reduce(0,:+).to_f/b[i].reduce(0,:+), 10)
+   end
+end
+def empty_sample
+   Hash[ALPHABET.map{|k| [k, []]}]
+end
+
+# Initialize
+$stderr.puts "Initializing." unless o[:q]
+sample_A = empty_sample
+sample_B = empty_sample
+last_label = nil
+prot_index = -1
+
+# Read file
+$stderr.puts "Reading input file." unless o[:q]
+ifh = File.open(o[:file], "r")
+ifh.each do |l|
+   r = l.chomp.split /\t/
+   if r.first != last_label
+      prot_index +=1
+      last_label = r.first
+      ALPHABET.each do |a|
+         sample_A[a][prot_index] = 0
+         sample_B[a][prot_index] = 0
+      end
+   end
+   [1,2].each do |ds|
+      unless %w(- *).include? r[ds]
+	 abort "Unknown amino acid in line #{$.}: '#{r[ds]}'." unless
+	    ALPHABET.include? r[ds]
+	 sample_A[ r[ds] ][ prot_index ] += 1 if ds==1
+	 sample_B[ r[ds] ][ prot_index ] += 1 if ds==2
+      end
+   end
+end
+ifh.close
+$stderr.puts "  > Found #{prot_index+1} proteins." unless o[:q]
+$stderr.puts "  > Saving #{o[:obs]}" unless o[:q]
+sum = dist_summary(sample_A, sample_B)
+File.open(o[:obs], "w") do |fh|
+   fh.puts ["AA", "log10_AB"].join("\t")
+   ALPHABET.each do |i|
+      fh.puts [i, sum.shift].join("\t")
+   end
+end
+
+# Permutations
+if File.size? o[:null] and not o[:overwrite]
+   $stderr.puts "Skipping permutations." unless o[:q]
+else
+   $stderr.puts "Permutating." unless o[:q]
+   permut_sum = []
+   o[:permutations].times do |i|
+      permut_A = empty_sample
+      permut_B = empty_sample
+      (0 .. prot_index).each do |j|
+	 # Copy counts of the protein
+	 ALPHABET.each do |k|
+	    permut_A[k][j] = sample_A[k][j]
+	    permut_B[k][j] = sample_B[k][j]
+	 end
+	 # Swap labels at random
+	 permut_A,permut_B = permut_B,permut_A if rand(2)==1
+      end
+      permut_sum << dist_summary(permut_A, permut_B)
+   end
+   $stderr.puts "  > Performed #{o[:permutations]} permutations." unless o[:q]
+   $stderr.puts "  > Saving #{o[:null]}" unless o[:q]
+   File.open(o[:null], "w") do |fh|
+      fh.puts ALPHABET.join("\t")
+      permut_sum.each{ |s| fh.puts s.join("\t") }
+   end
+end
+
+# Bootstraps
+if File.size? o[:boot] and not o[:overwrite]
+   $stderr.puts "Skipping bootstraps." unless o[:q]
+else
+   $stderr.puts "Bootstrapping." unless o[:q]
+   boot_sum = []
+   o[:bootstraps].times do |i|
+      boot_A = empty_sample
+      boot_B = empty_sample
+      (0 .. prot_index).each do |j|
+	 # Sample randomly with replacement
+	 jr = rand(prot_index+1)
+	 # Copy counts of the protein
+	 ALPHABET.each do |k|
+	    boot_A[k][j] = sample_A[k][jr]
+	    boot_B[k][j] = sample_B[k][jr]
+	 end
+      end
+      boot_sum << dist_summary(boot_A, boot_B)
+   end
+   $stderr.puts "  > Performed #{o[:bootstraps]} bootstraps." unless o[:q]
+   $stderr.puts "  > Saving #{o[:boot]}" unless o[:q]
+   File.open(o[:boot], "w") do |fh|
+      fh.puts ALPHABET.join("\t")
+      boot_sum.each{ |s| fh.puts s.join("\t") }
+   end
+end
+
+$stderr.puts "Done. Yayyy!" unless o[:q]