miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Pipelines/assembly.pbs/RUNME-2.bash

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+##################### HELP
+HELP="
+Usage:
+   $0 name[ prog[ k-mers]]
+   
+   name		The name of the run.  CONFIG.name.bash must exist.
+   prog		Program to execute.  One of 'soap' or 'velvet'.  By
+   		default, it executes both.
+   k-mers	Comma-separated list of k-mers to run.  By default,
+   		it executes all the odd numbers between 21 and 63
+		(inclusive).
+   
+   See $PDIR/README.txt for more information.
+"
+##################### RUN
+# Find the directory of the pipeline
+PDIR=$(dirname $(readlink -f $0));
+# Load variables
+source "$PDIR/RUNME.bash"
+if [[ "$SCRATCH" == "" ]] ; then
+  echo "$0: Error loading $PDIR/RUNME.bash, variable SCRATCH undefined" >&2
+  exit 1
+fi
+
+# Check request
+RUNVELVET=yes
+RUNSOAP=yes
+if [[ "$2" == "velvet" ]] ; then
+  RUNSOAP=no
+elif [[ "$2" == "soap" ]] ; then
+  RUNVELVET=no
+fi
+if [[ "$3" == "" ]] ; then
+  KMERARRAY="21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63"
+else
+  KMERARRAY=$3
+fi
+if [[ "$VELVETSIM" == "" ]] ; then
+  VELVETSIM=22
+fi
+if [[ "$SOAPSIM" == "" ]] ; then
+  let SOAPSIM=130/$PPN
+fi
+
+# Run it
+RAMMULT=${RAMMULT:-1}
+echo "Jobs being launched in $SCRATCH"
+for LIB in $LIBRARIES; do
+  # Prepare info
+  echo "Running $LIB";
+  if [[ "$USECOUPLED" == "yes" ]] ; then
+    INPUT="$DATA/$LIB.CoupledReads.fa"
+  elif [[ "$USESINGLE" == "yes" ]] ; then
+    INPUT="$DATA/$LIB.SingleReads.fa"
+  else
+    echo "$0: Error: No task selected, neither USECOUPLED nor USESINGLE set to yes." >&2
+    exit 1;
+  fi
+  VARS="LIB=$LIB,PDIR=$PDIR,DATA=$DATA,USECOUPLED=$USECOUPLED,USESINGLE=$USESINGLE"
+  [[ -n $INSLEN ]] && VARS="$VARS,INSLEN=$INSLEN"
+  [[ -n $VELVETG_EXTRA ]] && VARS="$VARS,VELVETG_EXTRA=$VELVETG_EXTRA"
+  [[ -n $VELVETH_EXTRA ]] && VARS="$VARS,VELVETH_EXTRA=$VELVETH_EXTRA"
+  [[ -n $CLEANUP ]] && VARS="$VARS,CLEANUP=$CLEANUP"
+  let SIZE=$(ls -lH "$INPUT" | awk '{print $5}')/1024/1024/1024;
+  let RAMS=40+$SIZE*10*$RAMMULT;
+  let RAMV=50+$SIZE*15*$RAMMULT;
+  # Launch Velvet
+  if [[ "$RUNVELVET" == "yes" ]] ; then
+    NAME="velvet_${LIB}"
+    if [[ "$QUEUE" != "" ]]; then
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l "walltime=$WTIME" -q "$QUEUE" \
+          -t "$KMERARRAY%$VELVETSIM"
+    elif [[ $RAMV -gt 150 ]]; then
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l walltime=360:00:00 -q biohimem-6 \
+          -t "$KMERARRAY%$VELVETSIM"
+    elif [[ $SIZE -lt 6 ]]; then
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l walltime=12:00:00 -q iw-shared-6 \
+          -t "$KMERARRAY%$VELVETSIM"
+    elif [[ $SIZE -lt 20 ]]; then
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l walltime=120:00:00 -q bioforce-6 \
+          -t "$KMERARRAY%$VELVETSIM"
+    else
+      qsub "$PDIR/velvet.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMV}gb -l walltime=360:00:00 -q biocluster-6 \
+          -t "$KMERARRAY%$VELVETSIM"
+    fi
+  fi
+  # Launch SOAP
+  if [[ "$RUNSOAP" == "yes" ]] ; then
+    NAME="soap_${LIB}"
+    if [[ "$QUEUE" != "" ]]; then
+      qsub "$PDIR/soap.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMS}gb -l walltime=$WTIME -q $QUEUE -l nodes=1:ppn=$PPN \
+          -t "$KMERARRAY%$SOAPSIM"
+    elif [[ $RAMS -gt 150 ]]; then
+      qsub "$PDIR/soap.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMS}gb -l walltime=48:00:00 -q biohimem-6 \
+          -l nodes=1:ppn=$PPN -t "$KMERARRAY%$SOAPSIM"
+    else
+      qsub "$PDIR/soap.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" \
+          -l mem=${RAMS}gb -l walltime=12:00:00 -q iw-shared-6 \
+          -l nodes=1:ppn=$PPN -t "$KMERARRAY%$SOAPSIM"
+    fi
+  fi
+done
+

data/utils/enveomics/Pipelines/assembly.pbs/RUNME-3.bash

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+##################### RUN
+# Find the directory of the pipeline
+PDIR=$(dirname $(readlink -f $0));
+# Load variables
+source "$PDIR/RUNME.bash"
+if [[ "$SCRATCH" == "" ]] ; then
+   echo "$0: Error loading $PDIR/RUNME.bash, variable SCRATCH undefined" >&2
+   exit 1
+fi
+
+# Run it
+echo "Jobs being launched in $SCRATCH"
+for LIB in $LIBRARIES; do
+   # Prepare info
+   echo "Running $LIB";
+   VARS="LIB=$LIB,PDIR=$PDIR"
+   # Launch Stats
+   NAME="N50_${LIB}"
+   qsub "$PDIR/stats.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME"
+done
+

data/utils/enveomics/Pipelines/assembly.pbs/RUNME-4.bash

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+##################### RUN
+# Find the directory of the pipeline
+PDIR=$(dirname $(readlink -f $0));
+# Load variables
+source "$PDIR/RUNME.bash"
+if [[ "$SCRATCH" == "" ]] ; then
+   echo "$0: Error loading $PDIR/RUNME.bash, variable SCRATCH undefined" >&2
+   exit 1
+fi
+
+# Run it
+echo "Jobs being launched in $SCRATCH"
+RAMMULT=${RAMMULT:-1}
+for LIB in $LIBRARIES; do
+   # Prepare info
+   echo "Running $LIB";
+   K_VELVET=$(echo $K_VELVET | sed -e 's/ /:/g')
+   K_SOAP=$(echo $K_SOAP | sed -e 's/ /:/g')
+   if [[ "$USECOUPLED" == "yes" ]] ; then
+      INPUT="$DATA/$LIB.CoupledReads.fa"
+   elif [[ "$USESINGLE" == "yes" ]] ; then
+      INPUT="$DATA/$LIB.SingleReads.fa"
+   else
+      echo "$0: Error: No task selected, neither USECOUPLED nor USESINGLE set to yes." >&2
+      exit 1;
+   fi
+   let SIZE=30*$(ls -l "$INPUT" | awk '{print $5}')/1024/1024/1024;
+   let RAM=\(3+$SIZE\)*$RAMMULT;
+   VARS="LIB=$LIB,PDIR=$PDIR,BIN454=$BIN454,KVELVET=$K_VELVET,KSOAP=$K_SOAP"
+   # Launch Newbler
+   NAME="Newbler_${LIB}"
+   if [[ "$QUEUE" != "" ]] ; then
+      qsub "$PDIR/newbler.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" -l nodes=1:ppn=$PPN -l mem=${RAM}g -l "walltime=$WTIME" -q "$QUEUE"
+   elif [[ $RAM -gt 150 ]] ; then
+      qsub "$PDIR/newbler.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" -l nodes=1:ppn=$PPN -l mem=${RAM}g -l walltime=360:00:00 -q biohimem-6
+   elif [[ $SIZE -lt 4 ]] ; then
+      qsub "$PDIR/newbler.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" -l nodes=1:ppn=$PPN -l mem=${RAM}g -l walltime=12:00:00 -q iw-shared-6
+   else
+      qsub "$PDIR/newbler.pbs" -v "$VARS" -d "$SCRATCH" -N "$NAME" -l nodes=1:ppn=$PPN -l mem=${RAM}g -l walltime=120:00:00 -q biocluster-6
+   fi
+done
+

data/utils/enveomics/Pipelines/assembly.pbs/RUNME.bash

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME-*.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone.  Execute one of RUNME-*.bash as described in the README.txt file" >&2
+   exit 1
+fi
+
+# Find the directory of the pipeline
+CWD=$(pwd)
+PDIR=$(dirname $(readlink -f $0));
+
+# Run it
+# Actually, this script doesn't run anything.  It's meant to keep the
+# variables centralized.
+
+# Load config
+NAMES=$(ls $PDIR/CONFIG.*.bash | sed -e 's/.*CONFIG\./    * /' | sed -e 's/\.bash//');
+if [[ "$1" == "" ]] ; then
+   if [[ "$HELP" == "" ]] ; then
+      echo "
+Usage:
+   $0 name
+   
+   name		The name of the run.  CONFIG.name.bash must exist.
+   
+   See $PDIR/README.txt for more information.
+   
+   Available names are:
+$NAMES
+" >&2
+   else
+      echo "$HELP   
+   Available names are:
+$NAMES
+" >&2
+   fi
+   exit 1
+fi
+if [[ ! -e "$PDIR/CONFIG.$1.bash" ]] ; then
+   echo "$0: Error: Impossible to find $PDIR/CONFIG.$1.bash, available names are:
+$NAMES" >&2
+   exit 1
+fi
+source "$PDIR/CONFIG.$1.bash"
+
+# Create the scratch directory
+if [[ ! -d $SCRATCH ]] ; then mkdir -p $SCRATCH ; fi;
+

data/utils/enveomics/Pipelines/assembly.pbs/kSelector.R

@@ -0,0 +1,37 @@
+
+# @author: Luis M. Rodriguez-R
+# @update: Nov-29-2012
+
+kSelector <- function(file, lib){
+   red <- rgb(0.6, 0, 0);
+   d <- read.table(file, sep=" ", h=T, fill=T);
+   d <- d[!is.na(d$N50) & !is.na(d$used), ];
+   d$reads <- max(d$reads, na.rm=T)
+   d <- d[order(d$K), ];
+   rownames(d) <- 1:nrow(d);
+   par(mar=c(5,4,4,5)+.1, cex=0.8);
+   barplot(d$reads/1e6, names=d$K, col='white', ylab='Number of reads (in millions)', xlab='K',
+      main=paste('Reads used and N50 by K-mers in the assembly of', lib));
+   barplot(d$used/1e6, col='grey', add=T);
+   par(new=T);
+   plot(1:length(d$K)-0.5, d$N50, col=red, t='b', lty=2, pch=20, cex=1, lwd=1.5,
+      xlim=c(0, length(d$K)), xaxt='n', yaxt='n', xlab='', ylab='');
+   axis(4, col.axis=red);
+   mtext('N50 (bp)', side=4, line=3, col=red);
+   # Suggest best k-mers
+   if(nrow(d) >= 3){
+      x = data.frame(K=d$K, l=(d$N50 - mean(d$N50))/sd(d$N50), u=(d$used - mean(d$used))/sd(d$used));
+      rownames(x) <- rownames(d)
+      d <- cbind(d, sel=FALSE);
+      k_s = c();
+      for(l_star in c(2, 1/2, 1)){
+         k_s_i = x$K[which.max(l_star*x$l + x$u)];
+	 k_s <- c(k_s, k_s_i);
+	 x <- x[x$K!=k_s_i, ];
+	 d$sel[d$K==k_s_i] <- TRUE;
+      }
+      abline(v=as.numeric(rownames(d)[d$sel])-0.5, col='darkgreen', lty=6);
+   }
+   return(d);
+}
+

data/utils/enveomics/Pipelines/assembly.pbs/newbler.pbs

@@ -0,0 +1,68 @@
+#!/bin/bash
+#PBS -l nodes=1:ppn=1
+#PBS -k oe
+
+# Some defaults for the parameters
+BIN454=${BIN454:-"$HOME/454/bin"};
+
+# Check mandatory variables
+if [[ "$LIB" == "" ]]; then
+   echo "Error: LIB is mandatory" >&2
+   exit 1;
+fi
+if [[ "$PDIR" == "" ]]; then
+   echo "Error: PDIR is mandatory" >&2
+   exit 1;
+fi
+if [[ "$KVELVET$KSOAP" == "" ]]; then
+   echo "Error: KVELVET and/or KSOAP are mandatory" >&2
+   exit 1;
+fi
+
+# Prepare input
+KVELVET=$(echo $KVELVET | sed -e 's/:/ /g')
+KSOAP=$(echo $KSOAP | sed -e 's/:/ /g')
+NP=$(cat "$PBS_NODEFILE" | wc -l)
+CWD=$(pwd)
+DIR="$CWD/$LIB.newbler"
+LOG="$DIR.log"
+module load perl/5.14.4
+export PATH=$PATH:$BIN454
+
+# Create project
+echo new > $DIR.proc
+nohup newAssembly $DIR > $LOG
+
+# Prepare Velvet
+if [[ "$KVELVET" != "" ]] ; then
+   echo pre-velvet > $DIR.proc
+   rm $LIB.velvet.tmp1 &>/dev/null
+   for K in $KVELVET ; do
+      perl "$PDIR/FastA.filterN.pl" "$LIB.velvet_$K/contigs.fa" >> $LIB.velvet.tmp1
+   done
+   perl "$PDIR/newbler_preparator.pl" $LIB.velvet.tmp1 $LIB.velvet.tmp2
+   cd $DIR
+   nohup addRun ../$LIB.velvet.tmp2 >> $LOG
+   cd $CWD
+fi ;
+
+# Prepare SOAP
+if [[ "$KSOAP" != "" ]] ; then
+   echo pre-soap > $DIR.proc
+   rm $LIB.soap.tmp1 &>/dev/null
+   for K in $KSOAP ; do
+      cat "$LIB.soap_$K/O.contig" >> $LIB.soap.tmp1
+   done
+   perl "$PDIR/newbler_preparator.pl" $LIB.soap.tmp1 $LIB.soap.tmp2
+   cd $DIR
+   nohup addRun ../$LIB.soap.tmp2 >> $LOG
+   cd $CWD
+fi ;
+
+# Run
+cd $DIR
+echo newbler > $DIR.proc
+nohup runProject -cpu $NP >> $LOG
+cd $CWD
+echo done > $DIR.proc
+

data/utils/enveomics/Pipelines/assembly.pbs/newbler_preparator.pl

@@ -0,0 +1,49 @@
+#!/usr/bin/perl
+
+
+my ($in, $out) = @ARGV;
+($in and $out) or die "
+Usage: $0 input.fa output.fa
+";
+
+open IN, "<", $in  or die "Cannot read file: $in: $!\n";
+open OUT,">", $out or die "Cannot create file: $out: $!\n";
+
+%reads=();
+@reads=();
+while(<IN>){
+  chomp;
+  if(/^\>/){
+	$tag=$_;
+	$reads{$tag}='';
+	push(@reads,$tag);
+  }else{
+	$reads{$tag}.=$_;
+  }
+}
+close(IN);
+
+for(0..$#reads){
+  $tag=$reads[$_];
+  $read=$reads{$tag};
+  $l=length $read;
+  if($l<100){
+	next;
+  }else{
+	if($l<1500){
+	  print OUT "$tag\n$read\n";
+	}else{
+	  $r=int($l/1500)+1;
+	  $start=0;
+	  $i=1;
+	  while($start<$l-100){
+		$tag_new=$tag.':r'.$i;
+		$i++;
+		$read_new=substr($read,$start,1500);
+		$start+=200;
+		print OUT "$tag_new\n$read_new\n";
+	  }
+	}
+  }
+}
+close(OUT);

data/utils/enveomics/Pipelines/assembly.pbs/soap.pbs

@@ -0,0 +1,80 @@
+#!/bin/bash
+#PBS -k oe
+
+# Some defaults for the parameters
+INSLEN=${INSLEN:-300};
+USECOUPLED=${USECOUPLED:-yes}
+USESINGLE=${USESINGLE:-no}
+CLEANUP=${CLEANUP:-yes}
+
+# Check mandatory variables
+if [[ "$LIB" == "" ]]; then
+   echo "Error: LIB is mandatory" >&2
+   exit 1;
+fi
+if [[ "$PDIR" == "" ]]; then
+   echo "Error: PDIR is mandatory" >&2
+   exit 1;
+fi
+if [[ "$DATA" == "" ]]; then
+   echo "Error: DATA is mandatory" >&2
+   exit 1;
+fi
+
+# Prepare input
+module load perl/5.14.4
+KMER=$PBS_ARRAYID
+DIR="$LIB.soap_$KMER"
+if [[ "$USECOUPLED" == "yes" ]]; then
+   MAXRDLEN=${MAXRDLEN:-$(perl $PDIR/FastA.length.pl "$DATA/$LIB.CoupledReads.fa" | head -n 200000 | awk '{if($2>MAX) MAX=$2} END{print MAX}')}
+elif [[ "$USESINGLE" == "yes" ]]; then
+   MAXRDLEN=${MAXRDLEN:-$(perl $PDIR/FastA.length.pl "$DATA/$LIB.SingleReads.fa" | head -n 200000 | awk '{if($2>MAX) MAX=$2} END{print MAX}')}
+else
+   echo "$0: Error: Nothing to do, neither USECOUPLED nor USESINGLE set to yes." >&2
+   exit 2
+fi
+NP=$(cat "$PBS_NODEFILE" | wc -l)
+
+# Config
+module load SOAP/denovo2/r240
+echo config > $DIR.proc
+echo "max_rd_len=$MAXRDLEN
+[LIB]
+reverse_seq=0
+asm_flag=3
+rank=1" > $DIR.config
+if [[ "$USECOUPLED" == "yes" ]]; then
+   echo "avg_ins=$INSLEN
+p=$DATA/$LIB.CoupledReads.fa" >> $DIR.config
+fi
+if [[ "$USESINGLE" == "yes" ]]; then
+   echo "f=$DATA/$LIB.SingleReads.fa" >> $DIR.config
+fi
+
+
+# Run
+echo pre > $DIR.proc
+if [[ -d $DIR ]] ; then rm -R $DIR ; fi
+mkdir $DIR
+echo soap > $DIR.proc
+SOAPdenovo-63mer all -s $DIR.config -p $NP -K $KMER -o $DIR/O &> $DIR.log
+if [[ -d $DIR ]] ; then
+   if [[ -s $DIR/O.contig ]] ; then
+      if [[ "$CLEANUP" != "no" ]] ; then
+	 echo cleanup > $DIR.proc
+	 rm $DIR/*edge
+	 rm $DIR/*vertex
+	 rm $DIR/*Arc*
+	 rm $DIR/*Graph*
+	 rm $DIR/*readInGap*
+      fi
+      echo done > $DIR.proc
+   else
+      echo "$0: Error: File $DIR/O.contig doesn't exist, something went wrong" >&2
+      exit 1
+   fi
+else
+   echo "$0: Error: Directory $DIR doesn't exist, something went wrong" >&2
+   exit 1
+fi
+