miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Pipelines/assembly.pbs/FastA.length.pl

@@ -0,0 +1,38 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M Rodriguez-R
+# @update  Oct-07-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+
+$#ARGV>=0 or die "
+Usage:
+   $0 seqs.fa... > length.txt
+
+   seqs.fa	One or more FastA files.
+   length.txt	A table with the lengths of the sequences.
+
+";
+
+for my $fa (@ARGV){
+   open FA, "<", $fa or die "Cannot open file: $fa: $!\n";
+   my $def = '';
+   my $len = 0;
+   while(<FA>){
+      next if /^;/;
+      if(m/^>(\S+)\s?/){
+         print "$def\t$len\n" if $def;
+	 $def = $1;
+	 $len = 0;
+      }else{
+         s/[^A-Za-z]//g;
+	 $len+= length $_;
+      }
+   }
+   print "$def\t$len\n" if $def;
+   close FA;
+}
+

data/utils/enveomics/Pipelines/assembly.pbs/README.md

@@ -0,0 +1,189 @@
+@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
+
+@update: Mar-17-2013
+
+@license: artistic 2.0
+
+@status: semi
+
+@pbs: yes
+
+# IMPORTANT
+
+This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
+are free to use it in other platforms with adequate adjustments.  It is largely
+based on Luo _et al._ 2012, ISME J.
+
+# PURPOSE
+
+This pipeline assemblies coupled and/or single reads from one or more libraries.
+It assumes that the reads have been quality-checked and trimmed.
+
+# HELP
+
+1. Files preparation:
+
+   1.1. Copy this folder to the cluster.
+   
+   1.2. Copy the sequences to the cluster.  Only trimmed/filtered reads are used.
+      All the files are expected to be in the same folder, and the filenames must
+      end in `.CoupledReads.fa` or `.SingleReads.fa`.
+   
+   1.3. Copy the file `CONFIG.mock.bash` to `CONFIG.<name>.bash`, where `<name>` is a
+      short name for your run (avoid characters other than alphanumeric).
+   
+   1.4. Change the variables in `CONFIG.<name>.bash`.  Notice that this pipeline
+      supports running several libraries at the same time, but it's strongly
+      recomended to run only one per config file, because the insert length
+      (in step 2) and the selected k-mers (in step 3) are fixed for all the
+      included libraries.  Also, there is a technical consideration:  The first
+      step will execute parallel jobs for each odd number between 21 and 63, and
+      SOAP will use 16 CPUs by default, which means 357 CPUs will be requested
+      per library in step 2.  It's a bad idea to run many libraries at the same
+      time.
+
+   1.5. If you have Mate-paired datasets (for example, prepared with Nextera), first
+      reverse-complement all the reads.  See also the `VELVETG_EXTRA` variable in
+      the `CONFIG.<name>.bash` file.
+
+2. Velvet and SOAP assembly:
+   
+   2.1. Execute `./RUNME-2.bash <name>` in the head node (see [troubleshooting](#troubleshooting) #1).
+   
+   2.2. Monitor the tasks named velvet_* and soap_*.
+   
+   2.3. Once completed, make sure the files .proc contain only the
+      word "done".  To do this, you may execute:
+```
+      grep -v '^done$' *.proc
+```
+
+      If successful, the output of the above command should be empty.  See
+      [Troubleshooting](#troubleshooting) #2 and #3 below if one or more of your jobs failed.
+
+3. K-mers selection:
+   
+   3.1. If you completed step 2, execute `./RUNME-3.bash <name>` in the head
+      node.
+   
+   3.2. Once completed, download and open the files `*.n50.pdf`.
+   
+   3.3. Select the three "best" k-mers for Velvet and for SOAP (they don't
+      have to be the same).  There is no well-tested method to select the
+      "best", and this is why this protocol is not automated, but semi-
+      automated.  A generally good rule-of-thumb is: pick one that optimizes
+      the amount of sequences used (these are the grey bars in the plot;
+      usually this is the smallest k-mer), pick one that optimizes the N50
+      (this is the dashed red line; usually this is a large k-mer), and pick
+      one that optimizes both (something in the middle).  You can select
+      more or less than three k-mers, this is just a suggestion.
+
+4. Newbler assembly:
+   
+   4.1. Edit the file `CONFIG.<name>.bash`: set the variables `K_VELVET` and
+      `K_SOAP` to contain the lists of "best" selected k-mers for Velvet and
+      SOAP, respectively.
+   
+   4.2. Execute `./RUNME-4.bash <name>` in the head node.
+   
+   4.3. Monitor the task newbler_*.  Once finished, your assembly is ready.
+      Once completed, make sure the file .newbler.proc contain only the
+      word "done".  To do this, you may execute:
+```
+      grep -v '^done$' *.proc
+```
+      If successful, the output should be empty.
+   
+   4.4. The final assembly should be located in the `SCRATCH` path, in a folder
+      named `<lib>.newbler/assembly/`.  The file `454AllContigs.fna` contains
+      all the assembled contigs, `454LargeContigs.fna` contains the contigs
+      with 500bp or more in length, and `454NewblerMetrics.txt` contains some
+      relevant statistics.
+
+
+# Comments
+
+* Some scripts contained in this package are actually symlinks to files in the
+  _Scripts_ folder.  Check the existance of these files when copied to
+  the cluster.
+
+# Troubleshooting
+
+1. Do I really have to change directory (`cd`) to the pipeline's folder everytime
+   I want to execute something?
+   
+   No.  Not really.  For simplicity, this file tells you to execute, for example,
+   `./RUNME-2.bash`.  However, you don't really have to be there, you can execute it
+   from any location.  For example, if you saved this pipeline in your home
+   directory, you can just execute `~/assembly.pbs/RUNME-2.bash` insted from any
+   location in the head node.
+
+2. I executed step 2, and Velvet worked but SOAP failed (or vice versa).  Can I
+   submit only one of them?
+
+   Yes.  To execute only Velvet, run:
+```
+   ./RUNME-2.bash <name> velvet
+```
+
+   To execute only SOAP, run:
+```
+   ./RUNME-2.bash <name> soap
+```
+
+3. I ran step 2, and most of the jobs finished, but few of them failed.  Can I
+   submit only few K-mers?
+
+   Yes.  To execute one kmer (say, the k-mer 33 of SOAP), run:
+```
+   ./RUNME-2.bash <name> soap 33
+```
+
+   You can also execute more than one kmer, using a comma-separated list.  For
+   example, to re-submit the k-mers 37, 39, and 41 of Velvet, run:
+```
+   ./RUNME-2.bash <name> velvet 37,39,41
+```
+
+4. What are the numbers on the job names of step 2?
+
+   The K-mer.  Each k-mer has it's own job, but they are "arrayed", to simplify
+   administration: notice that all the jobs of Velvet and all the jobs of SOAP
+   share the same job ID.
+
+5. Some jobs are being killed, why?
+
+   5.1. First, check the log file created by the pipeline.  The name is typically
+      the output prefix and the .log extension.  For velvet, there are two log files,
+      the `.glog` and the `.hlog`.  You may find the problem there.
+
+   5.2. Now, check the error file in your HOME directory.  The name depends on the
+      job, the library and the task.  For example: `~/soap_Mg_2-37.e1999838` is the
+      error file for step 2, task soap, library Mg_2, k-mer 37.  The appending
+      number after the 'e' is the job ID.  If this file contains errors probably
+      related to the pipeline, please let me know.
+
+   5.3. If you still have no clues, check the output file in your `HOME` directory.  The
+      name is just like the name of the error file (see #5.2 above), but with 'o'
+      instead of 'e'.  Compare the lines 'Resources' (what we asked the scheduler for)
+      and 'Rsrc Used' (what the job actually used).  A typical problem is that your
+      job may need more RAM than we asked for (the value of 'mem' in both lines).  If
+      the RAM used is larger than the RAM requested, the scheduler probably killed
+      your job.  To solve this, just go to your config file, and set the variable
+      RAMMULT to a number larger than 1.  For example, if you want to ask for double the
+      RAM, set `RAMMULT=2`.  You can also include simple arithmetic operations, like
+      `RAMMULT=3/2`.  If you want to add a fixed ammount of RAM, in Gib, use addition.
+      For example, to add 10G, set `RAMMULT=1+10`.
+
+   5.4.  Still no idea?  Try running the job again, sometimes the jobs fail with no
+      apparent reason, but they succeed when re-submited.  If your job keeps failing,
+      please gather as much information (the log, error and output files should be
+      enough) and let me take a look.
+
+6. In the step 2, some k-mers keep failing, and I just want to give up on them, can I?
+   
+   Yes.  Step 3 will analyze only completed jobs, so you can just ignore these faulty
+   k-mers.  Very small k-mers, for example, sometimes need too much memory, and very
+   large k-mers in Velvet sometimes need too much time.  If you don't think you're
+   missing too much, just ignore them.
+

data/utils/enveomics/Pipelines/assembly.pbs/RUNME-2.bash

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+##################### HELP
+HELP="
+Usage:
+   $0 name[ prog[ k-mers]]
+   
+   name		The name of the run.  CONFIG.name.bash must exist.
+   prog		Program to execute.  One of 'soap' or 'velvet'.  By
+   		default, it executes both.
+   k-mers	Comma-separated list of k-mers to run.  By default,
+   		it executes all the odd numbers between 21 and 63
+		(inclusive).
+   
+   See $PDIR/README.txt for more information.
+"
+##################### RUN
+# Find the directory of the pipeline
+PDIR=$(dirname $(readlink -f $0));
+# Load variables
+source "$PDIR/RUNME.bash"
+if [[ "$SCRATCH" == "" ]] ; then
+  echo "$0: Error loading $PDIR/RUNME.bash, variable SCRATCH undefined" >&2
+  exit 1
+fi
+
+# Check request
+RUNVELVET=yes
+RUNSOAP=yes
+if [[ "$2" == "velvet" ]] ; then
+  RUNSOAP=no
+elif [[ "$2" == "soap" ]] ; then
+  RUNVELVET=no
+fi
+if [[ "$3" == "" ]] ; then
+  KMERARRAY="21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63"
+else
+  KMERARRAY=$3
+fi
+if [[ "$VELVETSIM" == "" ]] ; then
+  VELVETSIM=22
+fi
+if [[ "$SOAPSIM" == "" ]] ; then
+  let SOAPSIM=130/$PPN
+fi
+
+# Run it
+RAMMULT=${RAMMULT:-1}
+echo "Jobs being launched in $SCRATCH"
+for LIB in $LIBRARIES; do
+  # Prepare info
+  echo "Running $LIB";
+  if [[ "$USECOUPLED" == "yes" ]] ; then
+    INPUT="$DATA/$LIB.CoupledReads.fa"
+  elif [[ "$USESINGLE" == "yes" ]] ; then
+    INPUT="$DATA/$LIB.SingleReads.fa"
+  else
+    echo "$0: Error: No task selected, neither USECOUPLED nor USESINGLE set to yes." >&2
+    exit 1;
+  fi
+  VARS="LIB=$LIB,PDIR=$PDIR,DATA=$DATA,USECOUPLED=$USECOUPLED,USESINGLE=$USESINGLE"
+  [[ -n $INSLEN ]] && VARS="$VARS,INSLEN=$INSLEN"
+  [[ -n $VELVETG_EXTRA ]] && VARS="$VARS,VELVETG_EXTRA=$VELVETG_EXTRA"
+  [[ -n $VELVETH_EXTRA ]] && VARS="$VARS,VELVETH_EXTRA=$VELVETH_EXTRA"
+  [[ -n $CLEANUP ]] && VARS="$VARS,CLEANUP=$CLEANUP"
+  let SIZE=$(ls -lH "$INPUT" | awk '{print $5}')/1024/1024/1024;
+  let RAMS=40+$SIZE*10*$RAMMULT;
+  let RAMV=50+$SIZE*15*$RAMMULT;
+  # Launch Velvet
+  if [[ "$RUNVELVET" == "yes" ]] ; then
+    NAME="velvet_${LIB}"
+    if [[ "$QUEUE" != "" ]]; then
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l "walltime=$WTIME" -q "$QUEUE" \
+          -t "$KMERARRAY%$VELVETSIM"
+    elif [[ $RAMV -gt 150 ]]; then
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l walltime=360:00:00 -q biohimem-6 \
+          -t "$KMERARRAY%$VELVETSIM"
+    elif [[ $SIZE -lt 6 ]]; then
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l walltime=12:00:00 -q iw-shared-6 \
+          -t "$KMERARRAY%$VELVETSIM"
+    elif [[ $SIZE -lt 20 ]]; then
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l walltime=120:00:00 -q bioforce-6 \
+          -t "$KMERARRAY%$VELVETSIM"
+    else
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l walltime=360:00:00 -q biocluster-6 \
+          -t "$KMERARRAY%$VELVETSIM"
+    fi
+  fi
+  # Launch SOAP
+  if [[ "$RUNSOAP" == "yes" ]] ; then
+    NAME="soap_${LIB}"
+    if [[ "$QUEUE" != "" ]]; then
+      qsub "$PDIR/soap.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMS}gb -l walltime=$WTIME -q $QUEUE -l nodes=1:ppn=$PPN \
+          -t "$KMERARRAY%$SOAPSIM"
+    elif [[ $RAMS -gt 150 ]]; then
+      qsub "$PDIR/soap.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMS}gb -l walltime=48:00:00 -q biohimem-6 \
+          -l nodes=1:ppn=$PPN -t "$KMERARRAY%$SOAPSIM"
+    else
+      qsub "$PDIR/soap.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMS}gb -l walltime=12:00:00 -q iw-shared-6 \
+          -l nodes=1:ppn=$PPN -t "$KMERARRAY%$SOAPSIM"
+    fi
+  fi
+done
+

data/utils/enveomics/Pipelines/assembly.pbs/RUNME-3.bash

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+##################### RUN
+# Find the directory of the pipeline
+PDIR=$(dirname $(readlink -f $0));
+# Load variables
+source "$PDIR/RUNME.bash"
+if [[ "$SCRATCH" == "" ]] ; then
+   echo "$0: Error loading $PDIR/RUNME.bash, variable SCRATCH undefined" >&2
+   exit 1
+fi
+
+# Run it
+echo "Jobs being launched in $SCRATCH"
+for LIB in $LIBRARIES; do
+   # Prepare info
+   echo "Running $LIB";
+   VARS="LIB=$LIB,PDIR=$PDIR"
+   # Launch Stats
+   NAME="N50_${LIB}"
+   qsub "$PDIR/stats.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME"
+done
+

data/utils/enveomics/Pipelines/assembly.pbs/RUNME-4.bash

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+##################### RUN
+# Find the directory of the pipeline
+PDIR=$(dirname $(readlink -f $0));
+# Load variables
+source "$PDIR/RUNME.bash"
+if [[ "$SCRATCH" == "" ]] ; then
+   echo "$0: Error loading $PDIR/RUNME.bash, variable SCRATCH undefined" >&2
+   exit 1
+fi
+
+# Run it
+echo "Jobs being launched in $SCRATCH"
+RAMMULT=${RAMMULT:-1}
+for LIB in $LIBRARIES; do
+   # Prepare info
+   echo "Running $LIB";
+   K_VELVET=$(echo $K_VELVET | sed -e 's/ /:/g')
+   K_SOAP=$(echo $K_SOAP | sed -e 's/ /:/g')
+   if [[ "$USECOUPLED" == "yes" ]] ; then
+      INPUT="$DATA/$LIB.CoupledReads.fa"
+   elif [[ "$USESINGLE" == "yes" ]] ; then
+      INPUT="$DATA/$LIB.SingleReads.fa"
+   else
+      echo "$0: Error: No task selected, neither USECOUPLED nor USESINGLE set to yes." >&2
+      exit 1;
+   fi
+   let SIZE=30*$(ls -l "$INPUT" | awk '{print $5}')/1024/1024/1024;
+   let RAM=\(3+$SIZE\)*$RAMMULT;
+   VARS="LIB=$LIB,PDIR=$PDIR,BIN454=$BIN454,KVELVET=$K_VELVET,KSOAP=$K_SOAP"
+   # Launch Newbler
+   NAME="Newbler_${LIB}"
+   if [[ "$QUEUE" != "" ]] ; then
+      qsub "$PDIR/newbler.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" -l nodes=1:ppn=$PPN -l mem=${RAM}g -l "walltime=$WTIME" -q "$QUEUE"
+   elif [[ $RAM -gt 150 ]] ; then
+      qsub "$PDIR/newbler.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" -l nodes=1:ppn=$PPN -l mem=${RAM}g -l walltime=360:00:00 -q biohimem-6
+   elif [[ $SIZE -lt 4 ]] ; then
+      qsub "$PDIR/newbler.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" -l nodes=1:ppn=$PPN -l mem=${RAM}g -l walltime=12:00:00 -q iw-shared-6
+   else
+      qsub "$PDIR/newbler.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" -l nodes=1:ppn=$PPN -l mem=${RAM}g -l walltime=120:00:00 -q biocluster-6
+   fi
+done
+

data/utils/enveomics/Pipelines/assembly.pbs/RUNME.bash

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME-*.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone.  Execute one of RUNME-*.bash as described in the README.txt file" >&2
+   exit 1
+fi
+
+# Find the directory of the pipeline
+CWD=$(pwd)
+PDIR=$(dirname $(readlink -f $0));
+
+# Run it
+# Actually, this script doesn't run anything.  It's meant to keep the
+# variables centralized.
+
+# Load config
+NAMES=$(ls $PDIR/CONFIG.*.bash | sed -e 's/.*CONFIG\./    * /' | sed -e 's/\.bash//');
+if [[ "$1" == "" ]] ; then
+   if [[ "$HELP" == "" ]] ; then
+      echo "
+Usage:
+   $0 name
+   
+   name		The name of the run.  CONFIG.name.bash must exist.
+   
+   See $PDIR/README.txt for more information.
+   
+   Available names are:
+$NAMES
+" >&2
+   else
+      echo "$HELP   
+   Available names are:
+$NAMES
+" >&2
+   fi
+   exit 1
+fi
+if [[ ! -e "$PDIR/CONFIG.$1.bash" ]] ; then
+   echo "$0: Error: Impossible to find $PDIR/CONFIG.$1.bash, available names are:
+$NAMES" >&2
+   exit 1
+fi
+source "$PDIR/CONFIG.$1.bash"
+
+# Create the scratch directory
+if [[ ! -d $SCRATCH ]] ; then mkdir -p $SCRATCH ; fi;
+