miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Pipelines/blast.pbs/TASK.check.bash

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone." >&2
+   echo "  Execute RUNME.bash as described in the README.txt file" >&2
+   exit 1
+fi
+
+# Check if the project exists
+if [[ ! -d "$SCRATCH" ]] ; then
+   echo "The project $PROJ doesn't exist at $SCRATCH_DIR." >&2
+   echo "  Execute '$PDIR/RUNME.bash $PROJ run' first." >&2
+   exit 1
+fi
+
+# Get log:
+echo "==[ Running tasks ]=="
+for i in $(ls $SCRATCH/log/status/* 2>/dev/null) ; do
+   echo "  $(basename $i): $(tail -n 1 $i)"
+done
+echo ""
+
+# Get active jobs:
+echo "==[ Active jobs ]=="
+job_r=0
+job_i=0
+job_c=0
+for i in $(ls $SCRATCH/log/active/* 2>/dev/null) ; do
+   jid=$(basename $i)
+   stat=$(qstat -f1 $jid 2>&1)
+   state=$(echo "$stat" | grep '^ *job_state = ' | sed -e 's/.*job_state = //')
+   case $state in
+   C)
+      code=$(echo "$stat" | grep '^ *exit_status = ' | sed -e 's/.*exit_status = //')
+      if [[ "$code" == "0" ]] ; then
+         mv "$i" "$SCRATCH/log/done/"
+	 let job_c=$job_c+1
+      else
+	 echo "Warning: Job $jid ($(cat $i|tr -d '\n')) failed with code $code." >&2
+	 echo "  see errors at: $(echo "$stat" | grep '^ *Error_Path = ' | sed -e 's/.*Error_Path = //')"
+         mv "$i" "$SCRATCH/log/failed/"
+      fi ;;
+   R)
+      echo "  Running: $jid: $(cat "$i")"
+      let job_r=$job_r+1 ;;
+   [HQW])
+      echo "  Idle: $jid: $(cat "$i")"
+      let job_i=$job_i+1 ;;
+   E)
+      echo "  Canceling: $jid: $(cat "$i")" ;;
+   *)
+      tmp_err=$(echo "$stat" | grep ERROR)
+      if [[ "$tmp_err" == "" ]] ; then
+	 echo "Warning: Unrecognized state: $jid: $state." >&2
+	 echo "  Please report this problem." >&2
+      else
+	 echo "  Error: $jid: $tmp_err"
+      fi ;;
+   esac
+   #subjobs=$(echo "$stat" | grep 'Sub-jobs:' | sed -e 's/.*: *//')
+   #if [[ "$subjobs" -gt 0 ]] ; then
+   #   echo "$stat" | grep '^ *\(Sub-jobs\|Active\|Eligible\|Blocked\|Completed\):' | sed -e 's/^ *//' | sed -e 's/  *//' | tr '\n' ' ' | sed -e 's/^/    /'
+   #   echo
+   #fi
+done
+if [[ $job_c -gt 0 ]] ; then
+   echo ""
+   echo "  Completed since last check: $job_c."
+fi
+if [[ $job_r -gt 0 || $job_i -gt 0 ]] ; then
+   echo ""
+   echo "  Running jobs: $job_r."
+   echo "  Idle jobs: $job_i."
+fi
+echo ""
+
+# Auto-trials
+echo "==[ Auto-trials ]=="
+if [[ -e "$SCRATCH/etc/trials" ]] ; then
+   trials=$(cat "$SCRATCH/etc/trials" | wc -l | sed -e 's/ //g')
+   if [[ $trials -gt 1 ]] ; then
+      echo "  $trials trials attempted:"
+   else
+      echo "  No recent failures in the current step, job launched:"
+   fi
+   cat "$SCRATCH/etc/trials" | sed -e 's/^/  o /' | sed -e 's/# $/No active trials\n/g'
+fi
+echo ""
+
+# Step-specific checks:
+echo "==[ Step summary ]=="
+todo=1
+if [[ -e "$SCRATCH/success/00" ]] ; then
+   echo "  Successful project initialization."
+   if [[ -e "$SCRATCH/success/01" ]] ; then
+      echo "  Successful input preparation."
+      if [[ -e "$SCRATCH/success/02" ]] ; then
+	 echo "  Successful BLAST execution."
+	 if [[ -e "$SCRATCH/success/02" ]] ; then
+	    echo "  Successful concatenation."
+	    echo "  Project finished successfully!"
+	    todo=0
+	 else
+	    echo "  Concatenating results."
+	 fi
+      else
+	 echo "  Running BLAST."
+      fi
+   else
+      echo "  Preparing input."
+   fi
+else
+   echo "  Initializing project."
+fi
+
+if [[ "$todo" -eq 1 && $job_r -eq 0 && $job_i -eq 0 ]] ; then
+   echo "  Job currently paused. To resume, execute:"
+   echo "  $PDIR/RUNME.bash $PROJ run"
+fi
+echo
+
+# Entire log
+echo "==[ Complete log ]=="
+for i in $(ls $SCRATCH/log/status/* 2>/dev/null) ; do
+   cat "$i" | sed -e "s/^/  $(basename $i): /"
+done

data/utils/enveomics/Pipelines/blast.pbs/TASK.dry.bash

@@ -0,0 +1,16 @@
+
+[[ "$QUEUE" == "" ]] && echo "Undefined QUEUE" >&2 && exit 1;
+[[ "$MAX_JOBS" == "" ]] && echo "Undefined MAX_JOBS" >&2 && exit 1;
+[[ "$PPN" == "" ]] && echo "Undefined PPN" >&2 && exit 1;
+[[ "$RAM" == "" ]] && echo "Undefined RAM" >&2 && exit 1;
+[[ "$SCRATCH_DIR" == "" ]] && echo "Undefined SCRATCH_DIR" >&2 && exit 1;
+[[ "$INPUT" == "" ]] && echo "Undefined INPUT" >&2 && exit 1;
+[[ "$DB" == "" ]] && echo "Undefined DB" >&2 && exit 1;
+[[ "$PROGRAM" == "" ]] && echo "Undefined PROGRAM" >&2 && exit 1;
+[[ "$MAX_TRIALS" == "" ]] && echo "Undefined MAX_TRIALS" >&2 && exit 1;
+[[ "$(type -t BEGIN)" == "function" ]] || ( echo "Undefined function BEGIN" && exit 1 ) ;
+[[ "$(type -t BEFORE_BLAST)" == "function" ]] || ( echo "Undefined function BEFORE_BLAST" && exit 1 ) ;
+[[ "$(type -t RUN_BLAST)" == "function" ]] || ( echo "Undefined function RUN_BLAST" && exit 1 ) ;
+[[ "$(type -t AFTER_BLAST)" == "function" ]] || ( echo "Undefined function AFTER_BLAST" && exit 1 ) ;
+[[ "$(type -t END)" == "function" ]] || ( echo "Undefined function END" && exit 1 ) ;
+

data/utils/enveomics/Pipelines/blast.pbs/TASK.eo.bash

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone." >&2
+   echo "  Execute RUNME.bash as described in the README.txt file" >&2 ;
+   exit 1 ;
+fi ;
+
+# Check if the project exists
+if [[ ! -d "$SCRATCH" ]] ; then
+   echo "The project $PROJ doesn't exist at $SCRATCH_DIR." >&2 ;
+   echo "  Execute '$PDIR/RUNME.bash $PROJ run' first." >&2 ;
+   exit 1 ;
+fi ;
+
+# Review errors
+(echo -e "==[ Last 10 lines of all e files ]==\nPress q to exit\n" ; tail -n 10 $SCRATCH/log/eo/*.e* ) | less
+# Review output
+(echo -e "==[ Last 100 lines of all o files ]==\nPress q to exit\n" ; tail -n 100 $SCRATCH/log/eo/*.o* ) | less
+

data/utils/enveomics/Pipelines/blast.pbs/TASK.pause.bash

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone." >&2
+   echo "  Execute RUNME.bash as described in the README.txt file" >&2 ;
+   exit 1 ;
+fi ;
+
+# Get active jobs:
+echo "======[ check ]======"
+job_r=0;
+job_i=0;
+job_c=0;
+
+echo "======[ pause ]======"
+for i in $(ls $SCRATCH/log/active/* 2>/dev/null) ; do
+   echo "  Pausing $jid." ;
+   jid=$(basename $i) ;
+   qdel $jid ;
+done ;
+
+# Restart auto-trials
+echo -n > "$SCRATCH/etc/trials" ;
+

data/utils/enveomics/Pipelines/blast.pbs/TASK.run.bash

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone.  Execute RUNME.bash as described in the README.txt file" >&2
+   exit 1
+fi
+
+# Check if too many auto-trials were attempted
+trials=0 ;
+if [[ -e "$SCRATCH/etc/trials" ]] ; then
+   trials=$(cat "$SCRATCH/etc/trials" | wc -l | sed -e 's/ //g');
+   if [[ $trials -ge $MAX_TRIALS ]] ; then
+      echo "The maximum number of trials was already attempted, halting." >&2 ;
+      exit 1 ;
+   fi ;
+fi ;
+
+# Create the scratch directory
+if [[ ! -d "$SCRATCH" ]] ; then mkdir -p "$SCRATCH" || exit 1 ; fi;
+
+if [[ ! -e "$SCRATCH/success/00" ]] ; then
+   # 00. Initialize the project
+   echo "00. Initializing project." >&2 ;
+   mkdir -p "$SCRATCH/tmp" "$SCRATCH/etc" "$SCRATCH/results" "$SCRATCH/success" || exit 1 ;
+   mkdir -p "$SCRATCH/log/active" "$SCRATCH/log/done" "$SCRATCH/log/failed" || exit 1 ;
+   mkdir -p "$SCRATCH/log/status" "$SCRATCH/log/eo" || exit 1 ;
+   echo "Preparing structure." >> "$SCRATCH/log/status/00" ;
+   # Build 01.bash
+   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM' -v '$MINVARS' -N '$PROJ-01' \\
+      '$PDIR/01.pbs.bash'|tr -d '\\n')" \
+      > "$SCRATCH/etc/01.bash" || exit 1 ;
+   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afterany:\$NEW_JOBID\" \\
+      -v \"$MINVARS,STEP=01,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-01-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
+      >> "$SCRATCH/etc/01.bash" || exit 1 ;
+   # Build 02.bash
+   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM,nodes=1:ppn=$PPN' \\
+      -v '$MINVARS' -N '$PROJ-02' -t '1-$MAX_JOBS' '$PDIR/02.pbs.bash'|tr -d '\\n')" \
+      > "$SCRATCH/etc/02.bash" \
+      || exit 1 ;
+   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afteranyarray:\$NEW_JOBID\" \\
+      -v \"$MINVARS,STEP=02,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-02-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
+      >> "$SCRATCH/etc/02.bash" \
+      || exit 1 ;
+   # Build 03.bash
+   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM' -v '$MINVARS' -N '$PROJ-03' \\
+      '$PDIR/03.pbs.bash'|tr -d '\\n')" \
+      > "$SCRATCH/etc/03.bash" || exit 1 ;
+   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afterany:\$NEW_JOBID\" \\
+      -v \"$MINVARS,STEP=03,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-03-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
+      >> "$SCRATCH/etc/03.bash" || exit 1 ;
+   
+   JOB_DONE "00" ;
+fi ;
+
+if [[ ! -e "$SCRATCH/success/01" ]] ; then
+   # 01. Preparing input
+   echo "01. Preparing input." >&2 ;
+   JOB01=$(LAUNCH_JOB "01" "00" "Preparing input files" "$SCRATCH/etc/01.bash") ;
+   echo "  New job: $JOB01." >&2 ;
+else
+   if [[ ! -e "$SCRATCH/success/02" ]] ; then
+      # 02. Launching BLAST
+      echo "02. Launching BLAST." >&2 ;
+      JOB02=$(LAUNCH_JOB "02" "00" "Running BLAST" "$SCRATCH/etc/02.bash") ;
+      echo "  New job: $JOB02." >&2 ;
+      # Clean on resubmission
+      cleaned=0
+      echo -n "  Cleaning completed sub-jobs on $JOB02: " >&2 ;
+      for jobi in $(seq 1 $MAX_JOBS) ; do
+	 if [[ -e "$SCRATCH/success/02.$jobi" ]] ; then
+	    qdel "$JOB02""[$jobi]" &> /dev/null ;
+	    let cleaned=$cleaned+1 ;
+	 fi ;
+      done ;
+      echo "$cleaned sub-jobs completed." >&2 ;
+   else
+      if [[ ! -e "$SCRATCH/success/03" ]] ; then
+	 # 03. Finalize
+	 echo "03. Finalizing." >&2 ;
+	 JOB03=$(LAUNCH_JOB "03" "00" "Concatenating results" "$SCRATCH/etc/03.bash") ;
+	 echo "  New job: $JOB03." >&2 ;
+      else
+         echo "Project complete, nothing to run." ;
+      fi ;
+   fi ;
+fi ;
+

data/utils/enveomics/Pipelines/blast.pbs/sentinel.pbs.bash

@@ -0,0 +1,29 @@
+# blast.pbs pipeline
+# Sentinel script
+
+echo "Sentinel script after $AFTERJOB" ;
+
+# Step-specific checks
+if [[ "$STEP" == "02" ]] ; then
+   # Read configuration
+   cd $SCRATCH ;
+   TASK="dry" ;
+   source "$PDIR/RUNME.bash" ;
+   
+   # Check tasks
+   INCOMPLETE=0;
+   for i in $(seq 1 $MAX_JOBS) ; do
+      if [[ ! -e "$SCRATCH/success/02.$i" ]] ; then
+	 let INCOMPLETE=$INCOMPLETE+1 ;
+      fi ;
+   done
+   if [[ $INCOMPLETE -eq 0 ]] ; then
+      JOB_DONE "02" ;
+   else
+      echo "$INCOMPLETE incomplete jobs, re-launching step 02." ;
+   fi ;
+fi
+
+# Continue the workflow
+"$PDIR/RUNME.bash" "$PROJ" run || exit 1 ;
+

data/utils/enveomics/Pipelines/idba.pbs/README.md

@@ -0,0 +1,49 @@
+@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
+
+@update: Feb-26-2015
+
+@license: artistic 2.0
+
+@status: auto
+
+@pbs: yes
+
+# IMPORTANT
+
+This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
+are free to use it in other platforms with adequate adjustments.
+
+# PURPOSE
+
+Performs assembly using IDBA-UD, designed for Single-Cell Genomics and Metagenomics.
+
+# HELP
+
+1. Files preparation:
+
+   1.1. Obtain the enveomics package in the cluster. You can use:
+      `git clone https://github.com/lmrodriguezr/enveomics.git`
+   
+   1.2. Prepare the trimmed reads (e.g., use trim.bs) in interposed FastA format. Files
+      must be raw, not zipped or packaged. Filenames must conform the format:
+      <name>.CoupledReads.fa, where <name> is the name of the sample. Locate all the
+      files within a folder named 04.trimmed_fasta, within your project folder. If you
+      used trim.pbs, no further action is necessary.
+   
+2. Pipeline execution:
+   
+   2.1. Simply execute `./RUNME.bash <dir> <data_type>`, where `<dir>` is the folder containing
+      the 04.trimmed_fasta folder, and `<data_type>` is a supported type of data (see help
+      message running `./RUNME.bash` without arguments).
+
+3. What to expect:
+
+   By the end of the run, you should find the folder *05.assembly*, including the following
+   files for each dataset:
+   
+   3.1. `<dataset>`: The IDBA output folder.
+   
+   3.2. `<dataset>.AllContigs.fna`: All contigs longer than 200bp in FastA format.
+
+   3.2. `<dataset>.LargeContigs.fna`: Contigs longer than 500bp in FastA format.
+

data/utils/enveomics/Pipelines/idba.pbs/RUNME.bash

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+if [[ "$1" == "" || "$1" == "-h" || "$2" == "" ]] ; then
+   echo "
+   Usage: ./RUNME.bash folder data_type [max_jobs]
+
+   folder	Path to the folder containing the 04.trimmed_fasta folder. The
+		trimmed reads must be in interposed FastA format, and filenames
+		must follow the format: <name>.CoupledReads.fa, where <name> is
+		the name of the sample. If non-paired, the filenames must follow
+		the format: <name>.SingleReads.fa. If both suffixes are found
+		for the same <name> prefix, they are both used.
+   data_type	Type of datasets in the project. One of: mg (for metagenomes),
+		scg (for single-cell genomes), g (for traditional genomes), or t
+		(for transcriptomes).
+   max_jobs	(optional) Maximum number of jobs to run in parallel. This
+		number can be increased, but bear in mind that this process is
+		highly I/O-intensive, and likely to crash or significantly slow
+		down the hard drive if many jobs are running simultaneously. By
+		default: 5.
+   " >&2
+   exit 1
+fi
+TYPE=$2
+if [[ "$TYPE" != "g" && "$TYPE" != "mg" && "$TYPE" != "scg" \
+		     && "$TYPE" != "t" ]] ; then
+   echo "Unsupported data type: $TYPE." >&2
+   exit 1
+fi
+if [[ "$3" == "" ]] ; then
+   MAX=5
+else
+   let MAX=$3+0
+fi
+
+dir=$(readlink -f $1)
+pac=$(dirname $(readlink -f $0))
+cwd=$(pwd)
+
+cd $dir
+if [[ ! -e 04.trimmed_fasta ]] ; then
+   echo "Cannot locate the 04.trimmed_fasta directory, aborting..." >&2
+   exit 1
+fi
+for i in 05.assembly ; do
+   [[ -d $i ]] || mkdir $i
+done
+
+k=0
+for i in $dir/04.trimmed_fasta/*.SingleReads.fa ; do
+   b=$(basename $i .SingleReads.fa)
+   touch $dir/04.trimmed_fasta/$b.CoupledReads.fa
+done
+
+for i in $dir/04.trimmed_fasta/*.CoupledReads.fa ; do
+   b=$(basename $i .CoupledReads.fa)
+   [[ -d $dir/05.assembly/$b ]] && continue
+   EXTRA=""
+   EXTRA_MSG=""
+   if [[ $k -ge $MAX ]] ; then
+      let prek=$k-$MAX
+      EXTRA="-W depend=afterany:${jids[$prek]}"
+      EXTRA_MSG=" (waiting for ${jids[$prek]})"
+   fi
+   
+   # Predict time (in hours)
+   SIZE_M=$(($(ls -pl 04.trimmed_fasta/$b.CoupledReads.fa \
+	       | awk '{print $5}')/1000000))
+   let TIME_H=6+$SIZE_M*2/1000
+   let RAM_G=20+$SIZE_M*20/1000
+   
+   # Find the right queue
+   if [[ $TIME_H -lt 12 ]] ; then
+      QUEUE="-q iw-shared-6 -l walltime=12:00:00"
+   elif [[ $TIME_H -lt 120 ]] ; then
+      QUEUE="-q microcluster -l walltime=120:00:00"
+   else
+      QUEUE="-q microcluster -l walltime=2000:00:00"
+   fi
+   
+   # Launch job
+   mkdir $dir/05.assembly/$b
+   OPTS="SAMPLE=$b,FOLDER=$dir,TYPE=$TYPE"
+   if [[ -s $dir/04.trimmed_fasta/$b.SingleReads.fa ]] ; then
+      OPTS="$OPTS,FA=$dir/04.trimmed_fasta/$b.SingleReads.fa"
+      [[ -s $dir/04.trimmed_fasta/$b.CoupledReads.fa ]] \
+	 && OPTS="$OPTS,FA_RL2=$dir/04.trimmed_fasta/$b.CoupledReads.fa"
+   else
+      OPTS="$OPTS,FA=$dir/04.trimmed_fasta/$b.CoupledReads.fa"
+   fi
+   jids[$k]=$(qsub -v "$OPTS" -N "IDBA-$b" -l "mem=${RAM_G}g" \
+	       $QUEUE $EXTRA $pac/run.pbs | grep .)
+   echo "$b: ${jids[$k]}$EXTRA_MSG"
+   let k=$k+1
+done