miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Pipelines/blast.pbs/RUNME.bash

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+##################### VARIABLES
+# Find the directory of the pipeline
+if [[ "$PDIR" == "" ]] ; then PDIR=$(dirname $(readlink -f $0)); fi ;
+CWD=$(pwd)
+
+# Load config
+if [[ "$PROJ" == "" ]] ; then PROJ="$1" ; fi
+if [[ "$TASK" == "" ]] ; then TASK="$2" ; fi
+if [[ "$TASK" == "" ]] ; then TASK="check" ; fi
+NAMES=$(ls $PDIR/CONFIG.*.bash | sed -e 's/.*CONFIG\./    o /' | sed -e 's/\.bash//');
+if [[ "$PROJ" == "" ]] ; then
+   if [[ "$HELP" == "" ]] ; then
+      echo "
+Usage:
+   $0 name task
+   
+   name	The name of the run.  CONFIG.name.bash must exist.
+   task	The action to perform.  One of:
+	o run: Executes the BLAST.
+	o check: Indicates the progress of the task (default).
+	o pause: Cancels running jobs (resume using run).
+	o dry: Checks that the parameters are correct, but doesn't run.
+	o eo: Review all eo files produced in the project.
+
+   See $PDIR/README.md for more information.
+   
+   Available names are:
+$NAMES
+" >&2
+   else
+      echo "$HELP   
+   Available names are:
+$NAMES
+" >&2
+   fi
+   exit 1
+fi
+if [[ ! -e "$PDIR/CONFIG.$PROJ.bash" ]] ; then
+   echo "$0: Error: Impossible to find $PDIR/CONFIG.$PROJ.bash, available names are:
+$NAMES" >&2
+   exit 1
+fi
+source "$PDIR/CONFIG.$PROJ.bash" ;
+SCRATCH="$SCRATCH_DIR/$PROJ" ;
+MINVARS="PDIR=$PDIR,SCRATCH=$SCRATCH,PROJ=$PROJ" ;
+case $QUEUE in
+bioforce-6)
+   MAX_H=120 ;;
+iw-shared-6)
+   MAX_H=12 ;;
+micro-largedata)
+   MAX_H=120 ;;
+biocluster-6 | biohimem-6 | microcluster)
+   MAX_H=240 ;;
+*)
+   echo "Unrecognized queue: $QUEUE." >&2 ;
+   exit 1 ;;
+esac ;
+
+##################### FUNCTIONS
+function REGISTER_JOB {
+   local STEP=$1
+   local SUBSTEP=$2
+   local MESSAGE=$3
+   local JOBID=$4
+
+   if [[ "$JOBID" != "" ]] ; then
+      MESSAGE="$MESSAGE [$JOBID]" ;
+      echo "$STEP: $SUBSTEP: $(date)" >> "$SCRATCH/log/active/$JOBID" ;
+   fi
+   echo "$MESSAGE." >> "$SCRATCH/log/status/$STEP" ;
+}
+
+function LAUNCH_JOB {
+   local STEP=$1
+   local SUBSTEP=$2
+   local MESSAGE=$3
+   local BASHFILE=$4
+   
+   cd "$SCRATCH/log/eo" ;
+   date >> "$SCRATCH/etc/trials" ;
+   source "$BASHFILE" || exit 1 ;
+   cd $CWD ;
+   if [[ "$SENTINEL_JOBID" != "" ]] ; then
+      REGISTER_JOB "$STEP" "$SUBSTEP" "Guarding job $NEW_JOBID" "$SENTINEL_JOBID" ;
+   fi ;
+   REGISTER_JOB "$STEP" "$SUBSTEP" "$MESSAGE" "$NEW_JOBID" ;
+   echo $NEW_JOBID ;
+}
+
+function JOB_DONE {
+   STEP=$1
+
+   echo "Done." >> "$SCRATCH/log/status/$STEP" ;
+   touch "$SCRATCH/success/$STEP" ;
+   echo -n '# ' > "$SCRATCH/etc/trials" ;
+}
+
+##################### RUN
+# Execute task
+if [[ ! -e "$PDIR/TASK.$TASK.bash" ]] ; then
+   echo "Unrecognized task: $TASK." >&2 ;
+   exit 1 ;
+else
+   source "$PDIR/TASK.$TASK.bash"
+fi
+

data/utils/enveomics/Pipelines/blast.pbs/TASK.check.bash

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone." >&2
+   echo "  Execute RUNME.bash as described in the README.txt file" >&2
+   exit 1
+fi
+
+# Check if the project exists
+if [[ ! -d "$SCRATCH" ]] ; then
+   echo "The project $PROJ doesn't exist at $SCRATCH_DIR." >&2
+   echo "  Execute '$PDIR/RUNME.bash $PROJ run' first." >&2
+   exit 1
+fi
+
+# Get log:
+echo "==[ Running tasks ]=="
+for i in $(ls $SCRATCH/log/status/* 2>/dev/null) ; do
+   echo "  $(basename $i): $(tail -n 1 $i)"
+done
+echo ""
+
+# Get active jobs:
+echo "==[ Active jobs ]=="
+job_r=0
+job_i=0
+job_c=0
+for i in $(ls $SCRATCH/log/active/* 2>/dev/null) ; do
+   jid=$(basename $i)
+   stat=$(qstat -f1 $jid 2>&1)
+   state=$(echo "$stat" | grep '^ *job_state = ' | sed -e 's/.*job_state = //')
+   case $state in
+   C)
+      code=$(echo "$stat" | grep '^ *exit_status = ' | sed -e 's/.*exit_status = //')
+      if [[ "$code" == "0" ]] ; then
+         mv "$i" "$SCRATCH/log/done/"
+	 let job_c=$job_c+1
+      else
+	 echo "Warning: Job $jid ($(cat $i|tr -d '\n')) failed with code $code." >&2
+	 echo "  see errors at: $(echo "$stat" | grep '^ *Error_Path = ' | sed -e 's/.*Error_Path = //')"
+         mv "$i" "$SCRATCH/log/failed/"
+      fi ;;
+   R)
+      echo "  Running: $jid: $(cat "$i")"
+      let job_r=$job_r+1 ;;
+   [HQW])
+      echo "  Idle: $jid: $(cat "$i")"
+      let job_i=$job_i+1 ;;
+   E)
+      echo "  Canceling: $jid: $(cat "$i")" ;;
+   *)
+      tmp_err=$(echo "$stat" | grep ERROR)
+      if [[ "$tmp_err" == "" ]] ; then
+	 echo "Warning: Unrecognized state: $jid: $state." >&2
+	 echo "  Please report this problem." >&2
+      else
+	 echo "  Error: $jid: $tmp_err"
+      fi ;;
+   esac
+   #subjobs=$(echo "$stat" | grep 'Sub-jobs:' | sed -e 's/.*: *//')
+   #if [[ "$subjobs" -gt 0 ]] ; then
+   #   echo "$stat" | grep '^ *\(Sub-jobs\|Active\|Eligible\|Blocked\|Completed\):' | sed -e 's/^ *//' | sed -e 's/  *//' | tr '\n' ' ' | sed -e 's/^/    /'
+   #   echo
+   #fi
+done
+if [[ $job_c -gt 0 ]] ; then
+   echo ""
+   echo "  Completed since last check: $job_c."
+fi
+if [[ $job_r -gt 0 || $job_i -gt 0 ]] ; then
+   echo ""
+   echo "  Running jobs: $job_r."
+   echo "  Idle jobs: $job_i."
+fi
+echo ""
+
+# Auto-trials
+echo "==[ Auto-trials ]=="
+if [[ -e "$SCRATCH/etc/trials" ]] ; then
+   trials=$(cat "$SCRATCH/etc/trials" | wc -l | sed -e 's/ //g')
+   if [[ $trials -gt 1 ]] ; then
+      echo "  $trials trials attempted:"
+   else
+      echo "  No recent failures in the current step, job launched:"
+   fi
+   cat "$SCRATCH/etc/trials" | sed -e 's/^/  o /' | sed -e 's/# $/No active trials\n/g'
+fi
+echo ""
+
+# Step-specific checks:
+echo "==[ Step summary ]=="
+todo=1
+if [[ -e "$SCRATCH/success/00" ]] ; then
+   echo "  Successful project initialization."
+   if [[ -e "$SCRATCH/success/01" ]] ; then
+      echo "  Successful input preparation."
+      if [[ -e "$SCRATCH/success/02" ]] ; then
+	 echo "  Successful BLAST execution."
+	 if [[ -e "$SCRATCH/success/02" ]] ; then
+	    echo "  Successful concatenation."
+	    echo "  Project finished successfully!"
+	    todo=0
+	 else
+	    echo "  Concatenating results."
+	 fi
+      else
+	 echo "  Running BLAST."
+      fi
+   else
+      echo "  Preparing input."
+   fi
+else
+   echo "  Initializing project."
+fi
+
+if [[ "$todo" -eq 1 && $job_r -eq 0 && $job_i -eq 0 ]] ; then
+   echo "  Job currently paused. To resume, execute:"
+   echo "  $PDIR/RUNME.bash $PROJ run"
+fi
+echo
+
+# Entire log
+echo "==[ Complete log ]=="
+for i in $(ls $SCRATCH/log/status/* 2>/dev/null) ; do
+   cat "$i" | sed -e "s/^/  $(basename $i): /"
+done

data/utils/enveomics/Pipelines/blast.pbs/TASK.dry.bash

@@ -0,0 +1,16 @@
+
+[[ "$QUEUE" == "" ]] && echo "Undefined QUEUE" >&2 && exit 1;
+[[ "$MAX_JOBS" == "" ]] && echo "Undefined MAX_JOBS" >&2 && exit 1;
+[[ "$PPN" == "" ]] && echo "Undefined PPN" >&2 && exit 1;
+[[ "$RAM" == "" ]] && echo "Undefined RAM" >&2 && exit 1;
+[[ "$SCRATCH_DIR" == "" ]] && echo "Undefined SCRATCH_DIR" >&2 && exit 1;
+[[ "$INPUT" == "" ]] && echo "Undefined INPUT" >&2 && exit 1;
+[[ "$DB" == "" ]] && echo "Undefined DB" >&2 && exit 1;
+[[ "$PROGRAM" == "" ]] && echo "Undefined PROGRAM" >&2 && exit 1;
+[[ "$MAX_TRIALS" == "" ]] && echo "Undefined MAX_TRIALS" >&2 && exit 1;
+[[ "$(type -t BEGIN)" == "function" ]] || ( echo "Undefined function BEGIN" && exit 1 ) ;
+[[ "$(type -t BEFORE_BLAST)" == "function" ]] || ( echo "Undefined function BEFORE_BLAST" && exit 1 ) ;
+[[ "$(type -t RUN_BLAST)" == "function" ]] || ( echo "Undefined function RUN_BLAST" && exit 1 ) ;
+[[ "$(type -t AFTER_BLAST)" == "function" ]] || ( echo "Undefined function AFTER_BLAST" && exit 1 ) ;
+[[ "$(type -t END)" == "function" ]] || ( echo "Undefined function END" && exit 1 ) ;
+

data/utils/enveomics/Pipelines/blast.pbs/TASK.eo.bash

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone." >&2
+   echo "  Execute RUNME.bash as described in the README.txt file" >&2 ;
+   exit 1 ;
+fi ;
+
+# Check if the project exists
+if [[ ! -d "$SCRATCH" ]] ; then
+   echo "The project $PROJ doesn't exist at $SCRATCH_DIR." >&2 ;
+   echo "  Execute '$PDIR/RUNME.bash $PROJ run' first." >&2 ;
+   exit 1 ;
+fi ;
+
+# Review errors
+(echo -e "==[ Last 10 lines of all e files ]==\nPress q to exit\n" ; tail -n 10 $SCRATCH/log/eo/*.e* ) | less
+# Review output
+(echo -e "==[ Last 100 lines of all o files ]==\nPress q to exit\n" ; tail -n 100 $SCRATCH/log/eo/*.o* ) | less
+

data/utils/enveomics/Pipelines/blast.pbs/TASK.pause.bash

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone." >&2
+   echo "  Execute RUNME.bash as described in the README.txt file" >&2 ;
+   exit 1 ;
+fi ;
+
+# Get active jobs:
+echo "======[ check ]======"
+job_r=0;
+job_i=0;
+job_c=0;
+
+echo "======[ pause ]======"
+for i in $(ls $SCRATCH/log/active/* 2>/dev/null) ; do
+   echo "  Pausing $jid." ;
+   jid=$(basename $i) ;
+   qdel $jid ;
+done ;
+
+# Restart auto-trials
+echo -n > "$SCRATCH/etc/trials" ;
+

data/utils/enveomics/Pipelines/blast.pbs/TASK.run.bash

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+##################### RUN
+# Check if it was sourced from RUNME.bash
+if [[ "$PDIR" == "" ]] ; then
+   echo "$0: Error: This file is not stand-alone.  Execute RUNME.bash as described in the README.txt file" >&2
+   exit 1
+fi
+
+# Check if too many auto-trials were attempted
+trials=0 ;
+if [[ -e "$SCRATCH/etc/trials" ]] ; then
+   trials=$(cat "$SCRATCH/etc/trials" | wc -l | sed -e 's/ //g');
+   if [[ $trials -ge $MAX_TRIALS ]] ; then
+      echo "The maximum number of trials was already attempted, halting." >&2 ;
+      exit 1 ;
+   fi ;
+fi ;
+
+# Create the scratch directory
+if [[ ! -d "$SCRATCH" ]] ; then mkdir -p "$SCRATCH" || exit 1 ; fi;
+
+if [[ ! -e "$SCRATCH/success/00" ]] ; then
+   # 00. Initialize the project
+   echo "00. Initializing project." >&2 ;
+   mkdir -p "$SCRATCH/tmp" "$SCRATCH/etc" "$SCRATCH/results" "$SCRATCH/success" || exit 1 ;
+   mkdir -p "$SCRATCH/log/active" "$SCRATCH/log/done" "$SCRATCH/log/failed" || exit 1 ;
+   mkdir -p "$SCRATCH/log/status" "$SCRATCH/log/eo" || exit 1 ;
+   echo "Preparing structure." >> "$SCRATCH/log/status/00" ;
+   # Build 01.bash
+   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM' -v '$MINVARS' -N '$PROJ-01' \\
+      '$PDIR/01.pbs.bash'|tr -d '\\n')" \
+      > "$SCRATCH/etc/01.bash" || exit 1 ;
+   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afterany:\$NEW_JOBID\" \\
+      -v \"$MINVARS,STEP=01,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-01-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
+      >> "$SCRATCH/etc/01.bash" || exit 1 ;
+   # Build 02.bash
+   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM,nodes=1:ppn=$PPN' \\
+      -v '$MINVARS' -N '$PROJ-02' -t '1-$MAX_JOBS' '$PDIR/02.pbs.bash'|tr -d '\\n')" \
+      > "$SCRATCH/etc/02.bash" \
+      || exit 1 ;
+   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afteranyarray:\$NEW_JOBID\" \\
+      -v \"$MINVARS,STEP=02,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-02-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
+      >> "$SCRATCH/etc/02.bash" \
+      || exit 1 ;
+   # Build 03.bash
+   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM' -v '$MINVARS' -N '$PROJ-03' \\
+      '$PDIR/03.pbs.bash'|tr -d '\\n')" \
+      > "$SCRATCH/etc/03.bash" || exit 1 ;
+   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afterany:\$NEW_JOBID\" \\
+      -v \"$MINVARS,STEP=03,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-03-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
+      >> "$SCRATCH/etc/03.bash" || exit 1 ;
+   
+   JOB_DONE "00" ;
+fi ;
+
+if [[ ! -e "$SCRATCH/success/01" ]] ; then
+   # 01. Preparing input
+   echo "01. Preparing input." >&2 ;
+   JOB01=$(LAUNCH_JOB "01" "00" "Preparing input files" "$SCRATCH/etc/01.bash") ;
+   echo "  New job: $JOB01." >&2 ;
+else
+   if [[ ! -e "$SCRATCH/success/02" ]] ; then
+      # 02. Launching BLAST
+      echo "02. Launching BLAST." >&2 ;
+      JOB02=$(LAUNCH_JOB "02" "00" "Running BLAST" "$SCRATCH/etc/02.bash") ;
+      echo "  New job: $JOB02." >&2 ;
+      # Clean on resubmission
+      cleaned=0
+      echo -n "  Cleaning completed sub-jobs on $JOB02: " >&2 ;
+      for jobi in $(seq 1 $MAX_JOBS) ; do
+	 if [[ -e "$SCRATCH/success/02.$jobi" ]] ; then
+	    qdel "$JOB02""[$jobi]" &> /dev/null ;
+	    let cleaned=$cleaned+1 ;
+	 fi ;
+      done ;
+      echo "$cleaned sub-jobs completed." >&2 ;
+   else
+      if [[ ! -e "$SCRATCH/success/03" ]] ; then
+	 # 03. Finalize
+	 echo "03. Finalizing." >&2 ;
+	 JOB03=$(LAUNCH_JOB "03" "00" "Concatenating results" "$SCRATCH/etc/03.bash") ;
+	 echo "  New job: $JOB03." >&2 ;
+      else
+         echo "Project complete, nothing to run." ;
+      fi ;
+   fi ;
+fi ;
+

data/utils/enveomics/Pipelines/blast.pbs/sentinel.pbs.bash

@@ -0,0 +1,29 @@
+# blast.pbs pipeline
+# Sentinel script
+
+echo "Sentinel script after $AFTERJOB" ;
+
+# Step-specific checks
+if [[ "$STEP" == "02" ]] ; then
+   # Read configuration
+   cd $SCRATCH ;
+   TASK="dry" ;
+   source "$PDIR/RUNME.bash" ;
+   
+   # Check tasks
+   INCOMPLETE=0;
+   for i in $(seq 1 $MAX_JOBS) ; do
+      if [[ ! -e "$SCRATCH/success/02.$i" ]] ; then
+	 let INCOMPLETE=$INCOMPLETE+1 ;
+      fi ;
+   done
+   if [[ $INCOMPLETE -eq 0 ]] ; then
+      JOB_DONE "02" ;
+   else
+      echo "$INCOMPLETE incomplete jobs, re-launching step 02." ;
+   fi ;
+fi
+
+# Continue the workflow
+"$PDIR/RUNME.bash" "$PROJ" run || exit 1 ;
+

data/utils/enveomics/Pipelines/idba.pbs/README.md

@@ -0,0 +1,49 @@
+@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
+
+@update: Feb-26-2015
+
+@license: artistic 2.0
+
+@status: auto
+
+@pbs: yes
+
+# IMPORTANT
+
+This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
+are free to use it in other platforms with adequate adjustments.
+
+# PURPOSE
+
+Performs assembly using IDBA-UD, designed for Single-Cell Genomics and Metagenomics.
+
+# HELP
+
+1. Files preparation:
+
+   1.1. Obtain the enveomics package in the cluster. You can use:
+      `git clone https://github.com/lmrodriguezr/enveomics.git`
+   
+   1.2. Prepare the trimmed reads (e.g., use trim.bs) in interposed FastA format. Files
+      must be raw, not zipped or packaged. Filenames must conform the format:
+      <name>.CoupledReads.fa, where <name> is the name of the sample. Locate all the
+      files within a folder named 04.trimmed_fasta, within your project folder. If you
+      used trim.pbs, no further action is necessary.
+   
+2. Pipeline execution:
+   
+   2.1. Simply execute `./RUNME.bash <dir> <data_type>`, where `<dir>` is the folder containing
+      the 04.trimmed_fasta folder, and `<data_type>` is a supported type of data (see help
+      message running `./RUNME.bash` without arguments).
+
+3. What to expect:
+
+   By the end of the run, you should find the folder *05.assembly*, including the following
+   files for each dataset:
+   
+   3.1. `<dataset>`: The IDBA output folder.
+   
+   3.2. `<dataset>.AllContigs.fna`: All contigs longer than 200bp in FastA format.
+
+   3.2. `<dataset>.LargeContigs.fna`: Contigs longer than 500bp in FastA format.
+