miga-base 1.2.17.1 → 1.2.17.3

data/utils/enveomics/Pipelines/blast.pbs/CONFIG.mock.bash

@@ -1,98 +0,0 @@
-#!/bin/bash
-
-##################### VARIABLES
-# Queue and resources.
-QUEUE="iw-shared-6" ;
-MAX_JOBS=500 ; # Maximum number of concurrent jobs. Never exceed 1990.
-PPN=2 ;
-RAM="9gb" ;
-
-# Paths
-SCRATCH_DIR="$HOME/scratch/pipelines/blast" ; # Where the outputs and temporals will be created
-INPUT="$HOME/data/my-large-file.fasta" ; # Input query file
-DB="$HOME/data/db/nr" ; # Input database
-PROGRAM="blastp" ;
-
-# Pipeline
-MAX_TRIALS=5 ; # Maximum number of automated attempts to re-start a job
-
-##################### FUNCTIONS
-## All the functions below can be edited to suit your particular job.
-## No function can be empty, but you can use a "dummy" function (like true).
-## All functions have access to any of the variables defined above.
-## 
-## The functions are executed in the following order (from left to right):
-##
-##           / -----> BEFORE_BLAST --> RUN_BLAST --> AFTER_BLAST ---\
-##          /              ···            ···            ···         \
-## BEGIN --#--------> BEFORE_BLAST --> RUN_BLAST --> AFTER_BLAST -----#---> END
-##          \              ···            ···            ···         /
-##           \ -----> BEFORE_BLAST --> RUN_BLAST --> AFTER_BLAST ---/
-##
-
-# Function to execute ONLY ONCE at the begining
-function BEGIN {
-   ### Format the database (assuming proteins, check commands):
-   # module load ncbi_blast/2.2.25 || exit 1 ;
-   # makeblastdb -in $HOME/data/some-database.faa -title $DB -dbtype prot || exit 1 ;
-   # module unload ncbi_blast/2.2.25 || exit 1 ;
-   ### Don't do anything:
-   true ;
-}
-
-# Function to execute BEFORE running the BLAST, for each sub-task.
-function BEFORE_BLAST {
-   local IN=$1 # Query file
-   local OUT=$2 # Blast file (to be created)
-   ### Don't do anything:
-   true ;
-}
-
-# Function that executes BLAST, for each sub-task
-function RUN_BLAST {
-   local IN=$1 # Query file
-   local OUT=$2 # Blast file (to be created)
-   ### Run BLAST+ with 13th and 14th columns (query length and subject length):
-   module load ncbi_blast/2.2.28_binary || exit 1 ;
-   $PROGRAM -query $IN -db $DB -out $OUT -num_threads $PPN \
-   	-outfmt "6 qseqid sseqid pident length mismatch gapopen qstart qend sstart send evalue bitscore qlen slen" \
-	|| exit 1 ;
-   module unload ncbi_blast/2.2.28_binary || exit 1 ;
-   ### Run BLAT (nucleotides)
-   # module load blat/rhel6 || exit 1 ;
-   # blat $DB $IN -out=blast8 $OUT || exit 1 ;
-   # module unload blat/rhel6 || exit 1 ;
-   ### Run BLAT (proteins)
-   # module load blat/rhel6 || exit 1 ;
-   # blat $DB $IN -out=blast8 -prot $OUT || exit 1 ;
-   # module unload blat/rhel6 || exit 1 ;
-}
-
-# Function to execute AFTER running the BLAST, for each sub-task
-function AFTER_BLAST {
-   local IN=$1 # Query files
-   local OUT=$2 # Blast files
-   ### Filter by best-match:
-   # sort $OUT | perl $PDIR/../../Scripts/BlastTab.best_hit_sorted.pl > $OUT.bm
-   ### Filter by Bit-score 60:
-   # awk '$12>=60' $OUT > $OUT.bs60
-   ### Filter by corrected identity 95 (only if it has the additional 13th column):
-   # awk '$3*$4/$13 >= 95' $OUT > $OUT.ci95
-   ### Don't do anything:
-   true ;
-}
-
-# Function to execute ONLY ONCE at the end, to concatenate the results
-function END {
-   local PREFIX=$1 # Prefix of all Blast files
-   local OUT=$2 # Single Blast output (to be created).
-   ### Simply concatenate files:
-   # cat $PREFIX.*.blast > $OUT
-   ### Concatenate only the filtered files (if filtering in AFTER_BLAST):
-   # cat $PREFIX.*.blast.bs60 > $OUT
-   ### Sort the BLAST by query (might require considerable RAM):
-   # sort -k 1 $PREFIX.*.blast > $OUT
-   ### Don't do anyhthing:
-   true ;
-}
-

data/utils/enveomics/Pipelines/blast.pbs/FastA.split.pl

@@ -1 +0,0 @@
-../../Scripts/FastA.split.pl

data/utils/enveomics/Pipelines/blast.pbs/README.md

@@ -1,127 +0,0 @@
-@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
-
-@update: Feb-20-2014
-
-@license: artistic 2.0
-
-@status: auto
-
-@pbs: yes
-
-# IMPORTANT
-
-This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
-are free to use it in other platforms with adequate adjustments.
-
-# PURPOSE
-
-Simplifies submitting and tracking large BLAST jobs in cluster.
-
-# HELP
-
-1. Files preparation:
-
-   1.1. Obtain the enveomics package in the cluster. You can use: `git clone https://github.com/lmrodriguezr/enveomics.git`
-   
-   1.2. Prepare the query sequences and the database.
-   
-   1.3. Copy the file `CONFIG.mock.bash` to `CONFIG.<name>.bash`, where `<name>` is a
-      short name for your project (avoid characters other than alphanumeric).
-   
-   1.4. Change the variables in `CONFIG.<name>.bash`. The **Queue and resources** and the
-      **Pipeline** variables are very standard, and can be kept unchanged. The **Paths**
-      variables indicate where your input files are and where the output files are to
-      be created, so check them carefully. Finally, the **FUNCTIONS** define the core
-      functionality of the pipeline, and should also be reviewed. By default, the
-      Pipeline simply runs BLAST+, with default parameters and tabular output with two
-      extra columns (qlen and slen). However, additional functionality can easily be
-      incorporated via these functions, such as BLAST filtering, concatenation, sorting,
-      or even execution of other programs instead of BLAST, such as BLAT, etc. Note that
-      the output MUST be BLAST-like tabular, because this is the only format supported
-      to check completeness and recover incomplete runs.
-   
-2. Pipeline execution:
-   
-   2.1. To initialize a run, execute: `./RUNME.bash <name> run`.
-
-   2.2. To check the status of a job, execute: `./RUNME.bash <name> check`.
-
-   2.3. To pause a run, execute: `./RUNME.bash <name> pause` (see 2.1 to resume).
-
-   2.4. To check if your CONFIG defines all required parameters, execute: `./RUNME.bash <name> dry`.
-
-   2.5. To review all the e/o files in the run, execute: `./RUNME.bash <name> eo`.
-
-3. Finalizing:
-   
-   3.1. `./RUNME.bash <name> check` will inform you if a project finished. If it finished successfully,
-      you can review your (split) results in $SCRATCH/results. If you concatenated the results in the
-      `END` function, you should have a file with all the results in $SCRATCH/<name>.blast.
-
-   3.2. Usually, checking the e/o files at the end is a good idea (`./RUNME.bash <name> eo`). However,
-      bear in mind that this Pipeline can overcome several errors and is robust to most failures, so
-      don't be alarmed at the first sight of errors.
-
-# Comments
-
-* Some scripts contained in this package are actually symlinks to files in the
-  _Scripts_ folder.  Check the existance of these files when copied to
-  the cluster.
-
-# Troubleshooting
-
-1. Do I really have to change directory (`cd`) to the pipeline's folder everytime I want to execute
-   something?
-   
-   No.  Not really.  For simplicity, this file tells you to execute `./RUNME.bash`.  However, you don't
-   really have to be there, you can execute it from any location.  For example, if you saved enveomics in
-   your home directory, you can just execute `~/enveomics/blast.pbs/RUNME.bash` insted from any location
-   in the head node.
-
-2. When I check a project, few sub-jobs are Active for much longer than the others. How do I know if those
-   sub-jobs are really active?
-   
-   Lets review an example of a problematic run. When you run `./RUNME.bash <name> check`, you see the
-   following in the "Active jobs" section:
-   ````
-   Idle: 155829.shared-sched.pace.gatech.edu: 02: 00: Mon Mar 17 14:10:28 EDT 2014
-     Sub-jobs:500 Active:4 ( 0.8% ) Eligible:0 ( 0.0% ) Blocked:0 ( 0.0% ) Completed:496 ( 99.2% ) 
-   Idle: 155830.shared-sched.pace.gatech.edu: 02: 00: Mon Mar 17 14:10:28 EDT 2014
-   
-   Running jobs: 0.
-   Idle jobs: 2.
-   ````
-   That means that the job "155829.shared-sched.pace.gatech.edu" has four Active jobs, while all the others are Completed. This is
-   a sign of something problematic.  You can see the complete status of each array using
-   `checkjob -v <JOB_NAME>`. In our example above, you would run `checkjob -v 155829`. In the output
-   of checkjob, most jobs should report "Completed". In this example, there are four jobs that are not
-   complete:
-   ````
-   ...
-   387 : 155829[387] : Completed
-   388 : 155829[388] : Running
-   389 : 155829[389] : Running
-   390 : 155829[390] : Running
-   391 : 155829[391] : Running
-   392 : 155829[392] : Completed
-   ...
-   ````
-   So you can simply check these sub-jobs in more detail. For example, if I run `checkjob -v 155829[388]`,
-   I see that the job is running in the machine `iw-k30-12.pace.gatech.edu` (Task Distribution), so I can try
-   to login to that machine to check if the job is actually running, using `top -u $(whoami)`. However, when
-   I run `ssh iw-k30-12`, I got a "Connection closed" error, which means that the machine hung up. At this point,
-   you might want to try one of the following solutions:
-
-   2.1. Pause the project using `./RUNME.bash <name> pause`, wait a few minutes, and resume using
-        `./RUNME.bash <name> run`. If you tried this a couple of times and you still have sub-jobs hanging, try:
-   
-   2.2. Check if your sub-jobs finished. Sometimes sub-jobs die too soon to return a success code, but they actually
-        finished. Just run the following command: `ls <SCRATCH>/<name>/success/02.* | wc -l`, where `<SCRATCH>` is the
-	value you set for the `SCRATCH` variable in the CONFIG file, and `<name>` is the name of your project. If the
-	output of that command is a number, and that number is exactly six times the number of jobs (`MAX_JOBS` in the
-	CONFIG file, typically 500), then your step 2 actually finished. In my case, I have 500 jobs, and the output
-	was 3000, so my job finished successfully, but the pipeline didn't notice. You can manually tell the pipeline
-	to go on running: `touch <SCRATCH>/<name>/success/02`, and pausing/resuming the project (see 2.1 above). If
-	the output is not the expected number (in my case, 3000, which is 6*500), DON'T RUN `touch`, just try the
-	solution 2.1 above once again.
-

data/utils/enveomics/Pipelines/blast.pbs/RUNME.bash

@@ -1,109 +0,0 @@
-#!/bin/bash
-
-##################### VARIABLES
-# Find the directory of the pipeline
-if [[ "$PDIR" == "" ]] ; then PDIR=$(dirname $(readlink -f $0)); fi ;
-CWD=$(pwd)
-
-# Load config
-if [[ "$PROJ" == "" ]] ; then PROJ="$1" ; fi
-if [[ "$TASK" == "" ]] ; then TASK="$2" ; fi
-if [[ "$TASK" == "" ]] ; then TASK="check" ; fi
-NAMES=$(ls $PDIR/CONFIG.*.bash | sed -e 's/.*CONFIG\./    o /' | sed -e 's/\.bash//');
-if [[ "$PROJ" == "" ]] ; then
-   if [[ "$HELP" == "" ]] ; then
-      echo "
-Usage:
-   $0 name task
-   
-   name	The name of the run.  CONFIG.name.bash must exist.
-   task	The action to perform.  One of:
-	o run: Executes the BLAST.
-	o check: Indicates the progress of the task (default).
-	o pause: Cancels running jobs (resume using run).
-	o dry: Checks that the parameters are correct, but doesn't run.
-	o eo: Review all eo files produced in the project.
-
-   See $PDIR/README.md for more information.
-   
-   Available names are:
-$NAMES
-" >&2
-   else
-      echo "$HELP   
-   Available names are:
-$NAMES
-" >&2
-   fi
-   exit 1
-fi
-if [[ ! -e "$PDIR/CONFIG.$PROJ.bash" ]] ; then
-   echo "$0: Error: Impossible to find $PDIR/CONFIG.$PROJ.bash, available names are:
-$NAMES" >&2
-   exit 1
-fi
-source "$PDIR/CONFIG.$PROJ.bash" ;
-SCRATCH="$SCRATCH_DIR/$PROJ" ;
-MINVARS="PDIR=$PDIR,SCRATCH=$SCRATCH,PROJ=$PROJ" ;
-case $QUEUE in
-bioforce-6)
-   MAX_H=120 ;;
-iw-shared-6)
-   MAX_H=12 ;;
-micro-largedata)
-   MAX_H=120 ;;
-biocluster-6 | biohimem-6 | microcluster)
-   MAX_H=240 ;;
-*)
-   echo "Unrecognized queue: $QUEUE." >&2 ;
-   exit 1 ;;
-esac ;
-
-##################### FUNCTIONS
-function REGISTER_JOB {
-   local STEP=$1
-   local SUBSTEP=$2
-   local MESSAGE=$3
-   local JOBID=$4
-
-   if [[ "$JOBID" != "" ]] ; then
-      MESSAGE="$MESSAGE [$JOBID]" ;
-      echo "$STEP: $SUBSTEP: $(date)" >> "$SCRATCH/log/active/$JOBID" ;
-   fi
-   echo "$MESSAGE." >> "$SCRATCH/log/status/$STEP" ;
-}
-
-function LAUNCH_JOB {
-   local STEP=$1
-   local SUBSTEP=$2
-   local MESSAGE=$3
-   local BASHFILE=$4
-   
-   cd "$SCRATCH/log/eo" ;
-   date >> "$SCRATCH/etc/trials" ;
-   source "$BASHFILE" || exit 1 ;
-   cd $CWD ;
-   if [[ "$SENTINEL_JOBID" != "" ]] ; then
-      REGISTER_JOB "$STEP" "$SUBSTEP" "Guarding job $NEW_JOBID" "$SENTINEL_JOBID" ;
-   fi ;
-   REGISTER_JOB "$STEP" "$SUBSTEP" "$MESSAGE" "$NEW_JOBID" ;
-   echo $NEW_JOBID ;
-}
-
-function JOB_DONE {
-   STEP=$1
-
-   echo "Done." >> "$SCRATCH/log/status/$STEP" ;
-   touch "$SCRATCH/success/$STEP" ;
-   echo -n '# ' > "$SCRATCH/etc/trials" ;
-}
-
-##################### RUN
-# Execute task
-if [[ ! -e "$PDIR/TASK.$TASK.bash" ]] ; then
-   echo "Unrecognized task: $TASK." >&2 ;
-   exit 1 ;
-else
-   source "$PDIR/TASK.$TASK.bash"
-fi
-

data/utils/enveomics/Pipelines/blast.pbs/TASK.check.bash

@@ -1,128 +0,0 @@
-#!/bin/bash
-
-##################### RUN
-# Check if it was sourced from RUNME.bash
-if [[ "$PDIR" == "" ]] ; then
-   echo "$0: Error: This file is not stand-alone." >&2
-   echo "  Execute RUNME.bash as described in the README.txt file" >&2
-   exit 1
-fi
-
-# Check if the project exists
-if [[ ! -d "$SCRATCH" ]] ; then
-   echo "The project $PROJ doesn't exist at $SCRATCH_DIR." >&2
-   echo "  Execute '$PDIR/RUNME.bash $PROJ run' first." >&2
-   exit 1
-fi
-
-# Get log:
-echo "==[ Running tasks ]=="
-for i in $(ls $SCRATCH/log/status/* 2>/dev/null) ; do
-   echo "  $(basename $i): $(tail -n 1 $i)"
-done
-echo ""
-
-# Get active jobs:
-echo "==[ Active jobs ]=="
-job_r=0
-job_i=0
-job_c=0
-for i in $(ls $SCRATCH/log/active/* 2>/dev/null) ; do
-   jid=$(basename $i)
-   stat=$(qstat -f1 $jid 2>&1)
-   state=$(echo "$stat" | grep '^ *job_state = ' | sed -e 's/.*job_state = //')
-   case $state in
-   C)
-      code=$(echo "$stat" | grep '^ *exit_status = ' | sed -e 's/.*exit_status = //')
-      if [[ "$code" == "0" ]] ; then
-         mv "$i" "$SCRATCH/log/done/"
-	 let job_c=$job_c+1
-      else
-	 echo "Warning: Job $jid ($(cat $i|tr -d '\n')) failed with code $code." >&2
-	 echo "  see errors at: $(echo "$stat" | grep '^ *Error_Path = ' | sed -e 's/.*Error_Path = //')"
-         mv "$i" "$SCRATCH/log/failed/"
-      fi ;;
-   R)
-      echo "  Running: $jid: $(cat "$i")"
-      let job_r=$job_r+1 ;;
-   [HQW])
-      echo "  Idle: $jid: $(cat "$i")"
-      let job_i=$job_i+1 ;;
-   E)
-      echo "  Canceling: $jid: $(cat "$i")" ;;
-   *)
-      tmp_err=$(echo "$stat" | grep ERROR)
-      if [[ "$tmp_err" == "" ]] ; then
-	 echo "Warning: Unrecognized state: $jid: $state." >&2
-	 echo "  Please report this problem." >&2
-      else
-	 echo "  Error: $jid: $tmp_err"
-      fi ;;
-   esac
-   #subjobs=$(echo "$stat" | grep 'Sub-jobs:' | sed -e 's/.*: *//')
-   #if [[ "$subjobs" -gt 0 ]] ; then
-   #   echo "$stat" | grep '^ *\(Sub-jobs\|Active\|Eligible\|Blocked\|Completed\):' | sed -e 's/^ *//' | sed -e 's/  *//' | tr '\n' ' ' | sed -e 's/^/    /'
-   #   echo
-   #fi
-done
-if [[ $job_c -gt 0 ]] ; then
-   echo ""
-   echo "  Completed since last check: $job_c."
-fi
-if [[ $job_r -gt 0 || $job_i -gt 0 ]] ; then
-   echo ""
-   echo "  Running jobs: $job_r."
-   echo "  Idle jobs: $job_i."
-fi
-echo ""
-
-# Auto-trials
-echo "==[ Auto-trials ]=="
-if [[ -e "$SCRATCH/etc/trials" ]] ; then
-   trials=$(cat "$SCRATCH/etc/trials" | wc -l | sed -e 's/ //g')
-   if [[ $trials -gt 1 ]] ; then
-      echo "  $trials trials attempted:"
-   else
-      echo "  No recent failures in the current step, job launched:"
-   fi
-   cat "$SCRATCH/etc/trials" | sed -e 's/^/  o /' | sed -e 's/# $/No active trials\n/g'
-fi
-echo ""
-
-# Step-specific checks:
-echo "==[ Step summary ]=="
-todo=1
-if [[ -e "$SCRATCH/success/00" ]] ; then
-   echo "  Successful project initialization."
-   if [[ -e "$SCRATCH/success/01" ]] ; then
-      echo "  Successful input preparation."
-      if [[ -e "$SCRATCH/success/02" ]] ; then
-	 echo "  Successful BLAST execution."
-	 if [[ -e "$SCRATCH/success/02" ]] ; then
-	    echo "  Successful concatenation."
-	    echo "  Project finished successfully!"
-	    todo=0
-	 else
-	    echo "  Concatenating results."
-	 fi
-      else
-	 echo "  Running BLAST."
-      fi
-   else
-      echo "  Preparing input."
-   fi
-else
-   echo "  Initializing project."
-fi
-
-if [[ "$todo" -eq 1 && $job_r -eq 0 && $job_i -eq 0 ]] ; then
-   echo "  Job currently paused. To resume, execute:"
-   echo "  $PDIR/RUNME.bash $PROJ run"
-fi
-echo
-
-# Entire log
-echo "==[ Complete log ]=="
-for i in $(ls $SCRATCH/log/status/* 2>/dev/null) ; do
-   cat "$i" | sed -e "s/^/  $(basename $i): /"
-done

data/utils/enveomics/Pipelines/blast.pbs/TASK.dry.bash

@@ -1,16 +0,0 @@
-
-[[ "$QUEUE" == "" ]] && echo "Undefined QUEUE" >&2 && exit 1;
-[[ "$MAX_JOBS" == "" ]] && echo "Undefined MAX_JOBS" >&2 && exit 1;
-[[ "$PPN" == "" ]] && echo "Undefined PPN" >&2 && exit 1;
-[[ "$RAM" == "" ]] && echo "Undefined RAM" >&2 && exit 1;
-[[ "$SCRATCH_DIR" == "" ]] && echo "Undefined SCRATCH_DIR" >&2 && exit 1;
-[[ "$INPUT" == "" ]] && echo "Undefined INPUT" >&2 && exit 1;
-[[ "$DB" == "" ]] && echo "Undefined DB" >&2 && exit 1;
-[[ "$PROGRAM" == "" ]] && echo "Undefined PROGRAM" >&2 && exit 1;
-[[ "$MAX_TRIALS" == "" ]] && echo "Undefined MAX_TRIALS" >&2 && exit 1;
-[[ "$(type -t BEGIN)" == "function" ]] || ( echo "Undefined function BEGIN" && exit 1 ) ;
-[[ "$(type -t BEFORE_BLAST)" == "function" ]] || ( echo "Undefined function BEFORE_BLAST" && exit 1 ) ;
-[[ "$(type -t RUN_BLAST)" == "function" ]] || ( echo "Undefined function RUN_BLAST" && exit 1 ) ;
-[[ "$(type -t AFTER_BLAST)" == "function" ]] || ( echo "Undefined function AFTER_BLAST" && exit 1 ) ;
-[[ "$(type -t END)" == "function" ]] || ( echo "Undefined function END" && exit 1 ) ;
-

data/utils/enveomics/Pipelines/blast.pbs/TASK.eo.bash

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-##################### RUN
-# Check if it was sourced from RUNME.bash
-if [[ "$PDIR" == "" ]] ; then
-   echo "$0: Error: This file is not stand-alone." >&2
-   echo "  Execute RUNME.bash as described in the README.txt file" >&2 ;
-   exit 1 ;
-fi ;
-
-# Check if the project exists
-if [[ ! -d "$SCRATCH" ]] ; then
-   echo "The project $PROJ doesn't exist at $SCRATCH_DIR." >&2 ;
-   echo "  Execute '$PDIR/RUNME.bash $PROJ run' first." >&2 ;
-   exit 1 ;
-fi ;
-
-# Review errors
-(echo -e "==[ Last 10 lines of all e files ]==\nPress q to exit\n" ; tail -n 10 $SCRATCH/log/eo/*.e* ) | less
-# Review output
-(echo -e "==[ Last 100 lines of all o files ]==\nPress q to exit\n" ; tail -n 100 $SCRATCH/log/eo/*.o* ) | less
-

data/utils/enveomics/Pipelines/blast.pbs/TASK.pause.bash

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-##################### RUN
-# Check if it was sourced from RUNME.bash
-if [[ "$PDIR" == "" ]] ; then
-   echo "$0: Error: This file is not stand-alone." >&2
-   echo "  Execute RUNME.bash as described in the README.txt file" >&2 ;
-   exit 1 ;
-fi ;
-
-# Get active jobs:
-echo "======[ check ]======"
-job_r=0;
-job_i=0;
-job_c=0;
-
-echo "======[ pause ]======"
-for i in $(ls $SCRATCH/log/active/* 2>/dev/null) ; do
-   echo "  Pausing $jid." ;
-   jid=$(basename $i) ;
-   qdel $jid ;
-done ;
-
-# Restart auto-trials
-echo -n > "$SCRATCH/etc/trials" ;
-

data/utils/enveomics/Pipelines/blast.pbs/TASK.run.bash

@@ -1,89 +0,0 @@
-#!/bin/bash
-
-##################### RUN
-# Check if it was sourced from RUNME.bash
-if [[ "$PDIR" == "" ]] ; then
-   echo "$0: Error: This file is not stand-alone.  Execute RUNME.bash as described in the README.txt file" >&2
-   exit 1
-fi
-
-# Check if too many auto-trials were attempted
-trials=0 ;
-if [[ -e "$SCRATCH/etc/trials" ]] ; then
-   trials=$(cat "$SCRATCH/etc/trials" | wc -l | sed -e 's/ //g');
-   if [[ $trials -ge $MAX_TRIALS ]] ; then
-      echo "The maximum number of trials was already attempted, halting." >&2 ;
-      exit 1 ;
-   fi ;
-fi ;
-
-# Create the scratch directory
-if [[ ! -d "$SCRATCH" ]] ; then mkdir -p "$SCRATCH" || exit 1 ; fi;
-
-if [[ ! -e "$SCRATCH/success/00" ]] ; then
-   # 00. Initialize the project
-   echo "00. Initializing project." >&2 ;
-   mkdir -p "$SCRATCH/tmp" "$SCRATCH/etc" "$SCRATCH/results" "$SCRATCH/success" || exit 1 ;
-   mkdir -p "$SCRATCH/log/active" "$SCRATCH/log/done" "$SCRATCH/log/failed" || exit 1 ;
-   mkdir -p "$SCRATCH/log/status" "$SCRATCH/log/eo" || exit 1 ;
-   echo "Preparing structure." >> "$SCRATCH/log/status/00" ;
-   # Build 01.bash
-   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM' -v '$MINVARS' -N '$PROJ-01' \\
-      '$PDIR/01.pbs.bash'|tr -d '\\n')" \
-      > "$SCRATCH/etc/01.bash" || exit 1 ;
-   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afterany:\$NEW_JOBID\" \\
-      -v \"$MINVARS,STEP=01,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-01-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
-      >> "$SCRATCH/etc/01.bash" || exit 1 ;
-   # Build 02.bash
-   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM,nodes=1:ppn=$PPN' \\
-      -v '$MINVARS' -N '$PROJ-02' -t '1-$MAX_JOBS' '$PDIR/02.pbs.bash'|tr -d '\\n')" \
-      > "$SCRATCH/etc/02.bash" \
-      || exit 1 ;
-   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afteranyarray:\$NEW_JOBID\" \\
-      -v \"$MINVARS,STEP=02,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-02-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
-      >> "$SCRATCH/etc/02.bash" \
-      || exit 1 ;
-   # Build 03.bash
-   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM' -v '$MINVARS' -N '$PROJ-03' \\
-      '$PDIR/03.pbs.bash'|tr -d '\\n')" \
-      > "$SCRATCH/etc/03.bash" || exit 1 ;
-   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afterany:\$NEW_JOBID\" \\
-      -v \"$MINVARS,STEP=03,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-03-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
-      >> "$SCRATCH/etc/03.bash" || exit 1 ;
-   
-   JOB_DONE "00" ;
-fi ;
-
-if [[ ! -e "$SCRATCH/success/01" ]] ; then
-   # 01. Preparing input
-   echo "01. Preparing input." >&2 ;
-   JOB01=$(LAUNCH_JOB "01" "00" "Preparing input files" "$SCRATCH/etc/01.bash") ;
-   echo "  New job: $JOB01." >&2 ;
-else
-   if [[ ! -e "$SCRATCH/success/02" ]] ; then
-      # 02. Launching BLAST
-      echo "02. Launching BLAST." >&2 ;
-      JOB02=$(LAUNCH_JOB "02" "00" "Running BLAST" "$SCRATCH/etc/02.bash") ;
-      echo "  New job: $JOB02." >&2 ;
-      # Clean on resubmission
-      cleaned=0
-      echo -n "  Cleaning completed sub-jobs on $JOB02: " >&2 ;
-      for jobi in $(seq 1 $MAX_JOBS) ; do
-	 if [[ -e "$SCRATCH/success/02.$jobi" ]] ; then
-	    qdel "$JOB02""[$jobi]" &> /dev/null ;
-	    let cleaned=$cleaned+1 ;
-	 fi ;
-      done ;
-      echo "$cleaned sub-jobs completed." >&2 ;
-   else
-      if [[ ! -e "$SCRATCH/success/03" ]] ; then
-	 # 03. Finalize
-	 echo "03. Finalizing." >&2 ;
-	 JOB03=$(LAUNCH_JOB "03" "00" "Concatenating results" "$SCRATCH/etc/03.bash") ;
-	 echo "  New job: $JOB03." >&2 ;
-      else
-         echo "Project complete, nothing to run." ;
-      fi ;
-   fi ;
-fi ;
-

data/utils/enveomics/Pipelines/blast.pbs/sentinel.pbs.bash

@@ -1,29 +0,0 @@
-# blast.pbs pipeline
-# Sentinel script
-
-echo "Sentinel script after $AFTERJOB" ;
-
-# Step-specific checks
-if [[ "$STEP" == "02" ]] ; then
-   # Read configuration
-   cd $SCRATCH ;
-   TASK="dry" ;
-   source "$PDIR/RUNME.bash" ;
-   
-   # Check tasks
-   INCOMPLETE=0;
-   for i in $(seq 1 $MAX_JOBS) ; do
-      if [[ ! -e "$SCRATCH/success/02.$i" ]] ; then
-	 let INCOMPLETE=$INCOMPLETE+1 ;
-      fi ;
-   done
-   if [[ $INCOMPLETE -eq 0 ]] ; then
-      JOB_DONE "02" ;
-   else
-      echo "$INCOMPLETE incomplete jobs, re-launching step 02." ;
-   fi ;
-fi
-
-# Continue the workflow
-"$PDIR/RUNME.bash" "$PROJ" run || exit 1 ;
-