miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Pipelines/assembly.pbs/kSelector.R

@@ -0,0 +1,37 @@
+
+# @author: Luis M. Rodriguez-R
+# @update: Nov-29-2012
+
+kSelector <- function(file, lib){
+   red <- rgb(0.6, 0, 0);
+   d <- read.table(file, sep=" ", h=T, fill=T);
+   d <- d[!is.na(d$N50) & !is.na(d$used), ];
+   d$reads <- max(d$reads, na.rm=T)
+   d <- d[order(d$K), ];
+   rownames(d) <- 1:nrow(d);
+   par(mar=c(5,4,4,5)+.1, cex=0.8);
+   barplot(d$reads/1e6, names=d$K, col='white', ylab='Number of reads (in millions)', xlab='K',
+      main=paste('Reads used and N50 by K-mers in the assembly of', lib));
+   barplot(d$used/1e6, col='grey', add=T);
+   par(new=T);
+   plot(1:length(d$K)-0.5, d$N50, col=red, t='b', lty=2, pch=20, cex=1, lwd=1.5,
+      xlim=c(0, length(d$K)), xaxt='n', yaxt='n', xlab='', ylab='');
+   axis(4, col.axis=red);
+   mtext('N50 (bp)', side=4, line=3, col=red);
+   # Suggest best k-mers
+   if(nrow(d) >= 3){
+      x = data.frame(K=d$K, l=(d$N50 - mean(d$N50))/sd(d$N50), u=(d$used - mean(d$used))/sd(d$used));
+      rownames(x) <- rownames(d)
+      d <- cbind(d, sel=FALSE);
+      k_s = c();
+      for(l_star in c(2, 1/2, 1)){
+         k_s_i = x$K[which.max(l_star*x$l + x$u)];
+	 k_s <- c(k_s, k_s_i);
+	 x <- x[x$K!=k_s_i, ];
+	 d$sel[d$K==k_s_i] <- TRUE;
+      }
+      abline(v=as.numeric(rownames(d)[d$sel])-0.5, col='darkgreen', lty=6);
+   }
+   return(d);
+}
+

data/utils/enveomics/Pipelines/assembly.pbs/newbler.pbs

@@ -0,0 +1,68 @@
+#!/bin/bash
+#PBS -l nodes=1:ppn=1
+#PBS -k oe
+
+# Some defaults for the parameters
+BIN454=${BIN454:-"$HOME/454/bin"};
+
+# Check mandatory variables
+if [[ "$LIB" == "" ]]; then
+   echo "Error: LIB is mandatory" >&2
+   exit 1;
+fi
+if [[ "$PDIR" == "" ]]; then
+   echo "Error: PDIR is mandatory" >&2
+   exit 1;
+fi
+if [[ "$KVELVET$KSOAP" == "" ]]; then
+   echo "Error: KVELVET and/or KSOAP are mandatory" >&2
+   exit 1;
+fi
+
+# Prepare input
+KVELVET=$(echo $KVELVET | sed -e 's/:/ /g')
+KSOAP=$(echo $KSOAP | sed -e 's/:/ /g')
+NP=$(cat "$PBS_NODEFILE" | wc -l)
+CWD=$(pwd)
+DIR="$CWD/$LIB.newbler"
+LOG="$DIR.log"
+module load perl/5.14.4
+export PATH=$PATH:$BIN454
+
+# Create project
+echo new > $DIR.proc
+nohup newAssembly $DIR > $LOG
+
+# Prepare Velvet
+if [[ "$KVELVET" != "" ]] ; then
+   echo pre-velvet > $DIR.proc
+   rm $LIB.velvet.tmp1 &>/dev/null
+   for K in $KVELVET ; do
+      perl "$PDIR/FastA.filterN.pl" "$LIB.velvet_$K/contigs.fa" >> $LIB.velvet.tmp1
+   done
+   perl "$PDIR/newbler_preparator.pl" $LIB.velvet.tmp1 $LIB.velvet.tmp2
+   cd $DIR
+   nohup addRun ../$LIB.velvet.tmp2 >> $LOG
+   cd $CWD
+fi ;
+
+# Prepare SOAP
+if [[ "$KSOAP" != "" ]] ; then
+   echo pre-soap > $DIR.proc
+   rm $LIB.soap.tmp1 &>/dev/null
+   for K in $KSOAP ; do
+      cat "$LIB.soap_$K/O.contig" >> $LIB.soap.tmp1
+   done
+   perl "$PDIR/newbler_preparator.pl" $LIB.soap.tmp1 $LIB.soap.tmp2
+   cd $DIR
+   nohup addRun ../$LIB.soap.tmp2 >> $LOG
+   cd $CWD
+fi ;
+
+# Run
+cd $DIR
+echo newbler > $DIR.proc
+nohup runProject -cpu $NP >> $LOG
+cd $CWD
+echo done > $DIR.proc
+

data/utils/enveomics/Pipelines/assembly.pbs/newbler_preparator.pl

@@ -0,0 +1,49 @@
+#!/usr/bin/perl
+
+
+my ($in, $out) = @ARGV;
+($in and $out) or die "
+Usage: $0 input.fa output.fa
+";
+
+open IN, "<", $in  or die "Cannot read file: $in: $!\n";
+open OUT,">", $out or die "Cannot create file: $out: $!\n";
+
+%reads=();
+@reads=();
+while(<IN>){
+  chomp;
+  if(/^\>/){
+	$tag=$_;
+	$reads{$tag}='';
+	push(@reads,$tag);
+  }else{
+	$reads{$tag}.=$_;
+  }
+}
+close(IN);
+
+for(0..$#reads){
+  $tag=$reads[$_];
+  $read=$reads{$tag};
+  $l=length $read;
+  if($l<100){
+	next;
+  }else{
+	if($l<1500){
+	  print OUT "$tag\n$read\n";
+	}else{
+	  $r=int($l/1500)+1;
+	  $start=0;
+	  $i=1;
+	  while($start<$l-100){
+		$tag_new=$tag.':r'.$i;
+		$i++;
+		$read_new=substr($read,$start,1500);
+		$start+=200;
+		print OUT "$tag_new\n$read_new\n";
+	  }
+	}
+  }
+}
+close(OUT);

data/utils/enveomics/Pipelines/assembly.pbs/soap.pbs

@@ -0,0 +1,80 @@
+#!/bin/bash
+#PBS -k oe
+
+# Some defaults for the parameters
+INSLEN=${INSLEN:-300};
+USECOUPLED=${USECOUPLED:-yes}
+USESINGLE=${USESINGLE:-no}
+CLEANUP=${CLEANUP:-yes}
+
+# Check mandatory variables
+if [[ "$LIB" == "" ]]; then
+   echo "Error: LIB is mandatory" >&2
+   exit 1;
+fi
+if [[ "$PDIR" == "" ]]; then
+   echo "Error: PDIR is mandatory" >&2
+   exit 1;
+fi
+if [[ "$DATA" == "" ]]; then
+   echo "Error: DATA is mandatory" >&2
+   exit 1;
+fi
+
+# Prepare input
+module load perl/5.14.4
+KMER=$PBS_ARRAYID
+DIR="$LIB.soap_$KMER"
+if [[ "$USECOUPLED" == "yes" ]]; then
+   MAXRDLEN=${MAXRDLEN:-$(perl $PDIR/FastA.length.pl "$DATA/$LIB.CoupledReads.fa" | head -n 200000 | awk '{if($2>MAX) MAX=$2} END{print MAX}')}
+elif [[ "$USESINGLE" == "yes" ]]; then
+   MAXRDLEN=${MAXRDLEN:-$(perl $PDIR/FastA.length.pl "$DATA/$LIB.SingleReads.fa" | head -n 200000 | awk '{if($2>MAX) MAX=$2} END{print MAX}')}
+else
+   echo "$0: Error: Nothing to do, neither USECOUPLED nor USESINGLE set to yes." >&2
+   exit 2
+fi
+NP=$(cat "$PBS_NODEFILE" | wc -l)
+
+# Config
+module load SOAP/denovo2/r240
+echo config > $DIR.proc
+echo "max_rd_len=$MAXRDLEN
+[LIB]
+reverse_seq=0
+asm_flag=3
+rank=1" > $DIR.config
+if [[ "$USECOUPLED" == "yes" ]]; then
+   echo "avg_ins=$INSLEN
+p=$DATA/$LIB.CoupledReads.fa" >> $DIR.config
+fi
+if [[ "$USESINGLE" == "yes" ]]; then
+   echo "f=$DATA/$LIB.SingleReads.fa" >> $DIR.config
+fi
+
+
+# Run
+echo pre > $DIR.proc
+if [[ -d $DIR ]] ; then rm -R $DIR ; fi
+mkdir $DIR
+echo soap > $DIR.proc
+SOAPdenovo-63mer all -s $DIR.config -p $NP -K $KMER -o $DIR/O &> $DIR.log
+if [[ -d $DIR ]] ; then
+   if [[ -s $DIR/O.contig ]] ; then
+      if [[ "$CLEANUP" != "no" ]] ; then
+	 echo cleanup > $DIR.proc
+	 rm $DIR/*edge
+	 rm $DIR/*vertex
+	 rm $DIR/*Arc*
+	 rm $DIR/*Graph*
+	 rm $DIR/*readInGap*
+      fi
+      echo done > $DIR.proc
+   else
+      echo "$0: Error: File $DIR/O.contig doesn't exist, something went wrong" >&2
+      exit 1
+   fi
+else
+   echo "$0: Error: Directory $DIR doesn't exist, something went wrong" >&2
+   exit 1
+fi
+

data/utils/enveomics/Pipelines/assembly.pbs/stats.pbs

@@ -0,0 +1,57 @@
+#!/bin/bash
+#PBS -q iw-shared-6
+#PBS -l nodes=1:ppn=1
+#PBS -l mem=1gb
+#PBS -l walltime=3:00:00
+#PBS -k oe
+
+# Check mandatory variables
+if [[ "$LIB" == "" ]]; then
+   echo "Error: LIB is mandatory" >&2
+   exit 1;
+fi
+if [[ "$PDIR" == "" ]]; then
+   echo "Error: PDIR is mandatory" >&2
+   exit 1;
+fi
+
+# Run
+module load perl/5.14.4
+echo "K N50 used reads " > $LIB.velvet.n50
+echo "K N50 used reads " > $LIB.soap.n50
+for ID in $(seq 10 31); do
+   let KMER=$ID*2+1
+   DIRV="$LIB.velvet_$KMER"
+   DIRS="$LIB.soap_$KMER"
+   echo $KMER > $LIB.velvet.n50.$KMER
+   echo $KMER > $LIB.soap.n50.$KMER
+   # N50 (>=500)
+   perl "$PDIR/FastA.N50.pl" "$DIRV/contigs.fa" 500 | grep '^N50' | sed -e 's/.*: //' >> $LIB.velvet.n50.$KMER
+   perl "$PDIR/FastA.N50.pl" "$DIRS/O.contig" 500 | grep '^N50' | sed -e 's/.*: //' >> $LIB.soap.n50.$KMER
+   # Used and Total reads
+   tail -n 1 $DIRV/Log | sed -e 's/.* using \\([0-9]*\\)\\/\\([0-9]*\\) reads.*/\\1\\n\\2/' >> $LIB.velvet.n50.$KMER
+   if [ -e "$DIRS/O.readOnContig" ] ; then
+      cat "$DIRS/O.readOnContig" | grep -vc '^read' >> $LIB.soap.n50.$KMER
+   elif [ -e "$DIRS/O.readOnContig.gz" ] ; then
+      zcat "$DIRS/O.readOnContig.gz" | grep -vc '^read' >> $LIB.soap.n50.$KMER
+   else
+      echo 0 >> $LIB.soap.n50.$KMER
+   fi
+   head -n 1 $DIRS/O.peGrads | awk '{print $3}' >> $LIB.soap.n50.$KMER
+   # Join
+   (cat $LIB.velvet.n50.$KMER | tr "\\n" " "; echo) >> $LIB.velvet.n50
+   rm $LIB.velvet.n50.$KMER
+   (cat $LIB.soap.n50.$KMER | tr "\\n" " "; echo) >> $LIB.soap.n50
+   rm $LIB.soap.n50.$KMER
+done
+
+# Create plot
+module load R/3.1.2
+echo "
+source('$PDIR/kSelector.R');
+pdf('$LIB.n50.pdf', 13, 7);
+kSelector('$LIB.velvet.n50', '$LIB (Velvet)');
+kSelector('$LIB.soap.n50', '$LIB (SOAP)');
+dev.off();
+" | R --vanilla -q
+

data/utils/enveomics/Pipelines/assembly.pbs/velvet.pbs

@@ -0,0 +1,63 @@
+#!/bin/bash
+#PBS -l nodes=1:ppn=1
+#PBS -k oe
+
+# Some defaults for the parameters
+FORMAT=${FORMAT:-fasta};
+INSLEN=${INSLEN:-300};
+USECOUPLED=${USECOUPLED:-yes};
+USESINGLE=${USESINGLE:-no};
+CLEANUP=${CLEANUP:-yes}
+
+# Check mandatory variables
+if [[ "$LIB" == "" ]]; then
+   echo "Error: LIB is mandatory" >&2
+   exit 1;
+fi
+if [[ "$PDIR" == "" ]]; then
+   echo "Error: PDIR is mandatory" >&2
+   exit 1;
+fi
+if [[ "$DATA" == "" ]]; then
+   echo "Error: DATA is mandatory" >&2
+   exit 1;
+fi
+
+# Prepare input
+KMER=$PBS_ARRAYID
+CWD=$(pwd)
+DIR="$CWD/$LIB.velvet_$KMER"
+
+# Run
+module load velvet/1.2.10
+echo velveth > $DIR.proc
+CMD="velveth_101_omp $DIR $KMER -$FORMAT"
+if [[ "$USECOUPLED" == "yes" ]]; then
+   CMD="$CMD -shortPaired $DATA/$LIB.CoupledReads.fa"
+fi
+if [[ "$USESINGLE" == "yes" ]]; then
+   CMD="$CMD -short $DATA/$LIB.SingleReads.fa"
+fi
+if [[ "$VELVETH_EXTRA" != "" ]]; then
+   CMD="$CMD $VELVETH_EXTRA"
+fi
+$CMD &> $DIR.hlog
+echo velvetg > $DIR.proc
+velvetg_101_omp "$DIR" -exp_cov auto -cov_cutoff auto -ins_length "$INSLEN" $VELVETG_EXTRA &> $DIR.glog
+if [[ -d $DIR ]] ; then
+   if [[ -s $DIR/contigs.fa ]] ; then
+      if [[ "$CLEANUP" != "no" ]] ; then
+	 echo cleanup > $DIR.proc
+	 rm $DIR/Sequences
+	 rm $DIR/Roadmaps
+	 rm $DIR/*Graph*
+      fi
+      echo done > $DIR.proc
+   else
+      echo "$0: Error: File $DIR/contigs.fa doesn't exist, something went wrong" >&2
+      exit 1
+   fi
+else
+   echo "$0: Error: Directory $DIR doesn't exist, something went wrong" >&2
+   exit 1
+fi

data/utils/enveomics/Pipelines/blast.pbs/01.pbs.bash

@@ -0,0 +1,38 @@
+# blast.pbs pipeline
+# Step 01 : Initialize input files
+
+# 00. Read configuration
+cd $SCRATCH ;
+TASK="dry" ;
+source "$PDIR/RUNME.bash" ;
+echo "$PBS_JOBID" > "$SCRATCH/success/01.00" ;
+
+if [[ ! -e "$SCRATCH/success/01.01" ]] ; then
+   # 01. BEGIN
+   REGISTER_JOB "01" "01" "Custom BEGIN function" \
+      && BEGIN \
+      || exit 1 ;
+   touch "$SCRATCH/success/01.01" ;
+fi
+
+if [[ ! -e "$SCRATCH/success/01.02" ]] ; then
+   # 02. Split
+   [[ -d "$SCRATCH/tmp/split" ]] && rm -R "$SCRATCH/tmp/split" ;
+   REGISTER_JOB "01" "02" "Splitting query files" \
+      && mkdir "$SCRATCH/tmp/split" \
+      && perl "$PDIR/FastA.split.pl" "$INPUT" "$SCRATCH/tmp/split/$PROJ" "$MAX_JOBS" \
+      || exit 1 ;
+   touch "$SCRATCH/success/01.02" ;
+fi ;
+
+if [[ ! -e "$SCRATCH/success/01.03" ]] ; then
+   # 03. Finalize
+   REGISTER_JOB "01" "03" "Finalizing input preparation" \
+      && mv "$SCRATCH/tmp/split" "$SCRATCH/tmp/in" \
+      || exit 1 ;
+   touch "$SCRATCH/success/01.03" ;
+fi ;
+
+[[ -d "$SCRATCH/tmp/out" ]] || ( mkdir "$SCRATCH/tmp/out" || exit 1 ) ;
+JOB_DONE "01" ;
+

data/utils/enveomics/Pipelines/blast.pbs/02.pbs.bash

@@ -0,0 +1,73 @@
+# blast.pbs pipeline
+# Step 02 : Run BLAST
+
+# Read configuration
+cd $SCRATCH ;
+TASK="dry" ;
+source "$PDIR/RUNME.bash" ;
+
+# 00. Initial vars
+ID_N=$PBS_ARRAYID
+[[ "$ID_N" == "" ]] && exit 1 ;
+[[ -e "$SCRATCH/success/02.$ID_N" ]] && exit 0 ;
+IN="$SCRATCH/tmp/in/$PROJ.$ID_N.fa" ;
+OUT="$SCRATCH/tmp/out/$PROJ.blast.$ID_N" ;
+FINAL_OUT="$SCRATCH/results/$PROJ.$ID_N.blast" ;
+if [[ -e "$SCRATCH/success/02.$ID_N.00" ]] ; then
+   pre_job=$(cat "$SCRATCH/success/02.$ID_N.00") ;
+   state=$(qstat -f "$pre_job" 2>/dev/null | grep job_state | sed -e 's/.*= //')
+   if [[ "$state" == "R" ]] ; then
+      echo "Warning: This task is already being executed by $pre_job. Aborting." >&2 ;
+      exit 0 ;
+   elif [[ "$state" == "" ]] ; then
+      echo "Warning: This task was initialized by $pre_job, but it's currently not running. Superseding." >&2 ;
+   fi ;
+fi
+echo "$PBS_JOBID" > "$SCRATCH/success/02.$ID_N.00" ;
+
+# 01. Before BLAST
+if [[ ! -e "$SCRATCH/success/02.$ID_N.01" ]] ; then
+   BEFORE_BLAST "$IN" "$OUT" || exit 1 ;
+   touch "$SCRATCH/success/02.$ID_N.01" ;
+fi ;
+
+# 02. Run BLAST
+if [[ ! -e "$SCRATCH/success/02.$ID_N.02" ]] ; then
+   # Recover previous runs, if any
+   if [[ -s "$OUT" ]] ; then
+      perl "$PDIR/BlastTab.recover_job.pl" "$IN" "$OUT" \
+	 || exit 1 ;
+   fi ;
+   # Run BLAST
+   RUN_BLAST "$IN" "$OUT" \
+      && mv "$OUT" "$OUT-z" \
+      || exit 1 ;
+   touch "$SCRATCH/success/02.$ID_N.02" ;
+fi ;
+
+# 03. Collect BLAST parts
+if [[ ! -e "$SCRATCH/success/02.$ID_N.03" ]] ; then
+   if [[ -e "$OUT" ]] ; then
+      echo "Warning: The file $OUT pre-exists, but the BLAST collection was incomplete." >&2 ;
+      echo "  I'm assuming that it corresponds to the first part of the result, but you should check manually." >&2 ;
+      echo "  The last lines are:" >&2 ;
+      tail -n 3 "$OUT" >&2 ;
+   else
+      touch "$OUT" || exit 1 ;
+   fi ;
+   for i in $(ls $OUT-*) ; do
+      cat "$i" >> "$OUT" ;
+      rm "$i" || exit 1 ;
+   done ;
+   mv "$OUT" "$FINAL_OUT"
+   touch "$SCRATCH/success/02.$ID_N.03" ;
+fi ;
+
+# 04. After BLAST
+if [[ ! -e "$SCRATCH/success/02.$ID_N.04" ]] ; then
+   AFTER_BLAST "$IN" "$FINAL_OUT" || exit 1 ;
+   touch "$SCRATCH/success/02.$ID_N.04" ;
+fi ;
+
+touch "$SCRATCH/success/02.$ID_N" ;
+