miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Pipelines/blast.pbs/03.pbs.bash

@@ -0,0 +1,21 @@
+# blast.pbs pipeline
+# Step 03 : Finalize
+
+# Read configuration
+cd $SCRATCH ;
+TASK="dry" ;
+source "$PDIR/RUNME.bash" ;
+PREFIX="$SCRATCH/results/$PROJ" ;
+OUT="$SCRATCH/$PROJ.blast" ;
+echo "$PBS_JOBID" > "$SCRATCH/success/02.00" ;
+
+# 01. END
+if [[ ! -e "$SCRATCH/success/03.01" ]] ; then
+   REGISTER_JOB "03" "01" "Custom END function" \
+      && END "$PREFIX" "$OUT" \
+      || exit 1 ;
+   touch "$SCRATCH/success/03.01" ;
+fi ;
+
+JOB_DONE "03" ;
+

data/utils/enveomics/Pipelines/blast.pbs/BlastTab.recover_job.pl

@@ -0,0 +1,72 @@
+#!/usr/bin/perl
+
+use warnings;
+use strict;
+use File::Copy;
+
+my($fasta, $blast) = @ARGV;
+
+($fasta and $blast) or die "
+.USAGE:
+   $0 query.fa blast.txt
+
+   query.fa	Query sequences in FastA format.
+   blast.txt	Incomplete BLAST output in tabular format.
+
+";
+
+print "Fixing $blast:\n";
+my $blast_res;
+for(my $i=0; 1; $i++){
+   $blast_res = "$blast-$i";
+   last unless -e $blast_res;
+}
+open BLAST, "<", $blast or die "Cannot read the file: $blast: $!\n";
+open TMP, ">", "$blast-tmp" or die "Cannot create the file: $blast-tmp: $!\n";
+my $last="";
+my $last_id="";
+my $before = "";
+while(my $ln=<BLAST>){
+   chomp $ln;
+   last unless $ln =~ m/(.+?)\t/;
+   my $id = $1;
+   if($id eq $last_id){
+      $last.= $ln."\n";
+   }else{
+      print TMP $last if $last;
+      $before = $last_id;
+      $last = $ln."\n";
+      $last_id = $id;
+   }
+}
+close BLAST;
+close TMP;
+
+move "$blast-tmp", $blast_res or die "Cannot move file $blast-tmp into $blast_res: $!\n";
+unlink $blast or die "Cannot delete file: $blast: $!\n";
+
+unless($before eq ""){
+   print "[$before] ";
+   $before = ">$before";
+   
+   open FASTA, "<", $fasta or die "Cannot read file: $fasta: $!\n";
+   open TMP, ">", "$fasta-tmp" or die "Cannot create file: $fasta-tmp: $!\n";
+   my $print = 0;
+   my $at = 0;
+   my $i = 0;
+   while(my $ln=<FASTA>){
+      $i++;
+      $print = 1 if $at and $ln =~ /^>/;
+      print TMP $ln if $print;
+      $ln =~ s/\s+.*//;
+      chomp $ln;
+      $at = $i if $ln eq $before;
+   }
+   close TMP;
+   close FASTA;
+   printf 'recovered at %.2f%% (%d/%d).'."\n", 100*$at/$i, $at, $i if $i;
+
+   move $fasta, "$fasta.old" or die "Cannot move file $fasta into $fasta.old: $!\n";
+   move "$fasta-tmp", $fasta or die "Cannot move file $fasta-tmp into $fasta: $!\n";
+}
+

data/utils/enveomics/Pipelines/blast.pbs/CONFIG.mock.bash

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+##################### VARIABLES
+# Queue and resources.
+QUEUE="iw-shared-6" ;
+MAX_JOBS=500 ; # Maximum number of concurrent jobs. Never exceed 1990.
+PPN=2 ;
+RAM="9gb" ;
+
+# Paths
+SCRATCH_DIR="$HOME/scratch/pipelines/blast" ; # Where the outputs and temporals will be created
+INPUT="$HOME/data/my-large-file.fasta" ; # Input query file
+DB="$HOME/data/db/nr" ; # Input database
+PROGRAM="blastp" ;
+
+# Pipeline
+MAX_TRIALS=5 ; # Maximum number of automated attempts to re-start a job
+
+##################### FUNCTIONS
+## All the functions below can be edited to suit your particular job.
+## No function can be empty, but you can use a "dummy" function (like true).
+## All functions have access to any of the variables defined above.
+## 
+## The functions are executed in the following order (from left to right):
+##
+##           / -----> BEFORE_BLAST --> RUN_BLAST --> AFTER_BLAST ---\
+##          /              ···            ···            ···         \
+## BEGIN --#--------> BEFORE_BLAST --> RUN_BLAST --> AFTER_BLAST -----#---> END
+##          \              ···            ···            ···         /
+##           \ -----> BEFORE_BLAST --> RUN_BLAST --> AFTER_BLAST ---/
+##
+
+# Function to execute ONLY ONCE at the begining
+function BEGIN {
+   ### Format the database (assuming proteins, check commands):
+   # module load ncbi_blast/2.2.25 || exit 1 ;
+   # makeblastdb -in $HOME/data/some-database.faa -title $DB -dbtype prot || exit 1 ;
+   # module unload ncbi_blast/2.2.25 || exit 1 ;
+   ### Don't do anything:
+   true ;
+}
+
+# Function to execute BEFORE running the BLAST, for each sub-task.
+function BEFORE_BLAST {
+   local IN=$1 # Query file
+   local OUT=$2 # Blast file (to be created)
+   ### Don't do anything:
+   true ;
+}
+
+# Function that executes BLAST, for each sub-task
+function RUN_BLAST {
+   local IN=$1 # Query file
+   local OUT=$2 # Blast file (to be created)
+   ### Run BLAST+ with 13th and 14th columns (query length and subject length):
+   module load ncbi_blast/2.2.28_binary || exit 1 ;
+   $PROGRAM -query $IN -db $DB -out $OUT -num_threads $PPN \
+   	-outfmt "6 qseqid sseqid pident length mismatch gapopen qstart qend sstart send evalue bitscore qlen slen" \
+	|| exit 1 ;
+   module unload ncbi_blast/2.2.28_binary || exit 1 ;
+   ### Run BLAT (nucleotides)
+   # module load blat/rhel6 || exit 1 ;
+   # blat $DB $IN -out=blast8 $OUT || exit 1 ;
+   # module unload blat/rhel6 || exit 1 ;
+   ### Run BLAT (proteins)
+   # module load blat/rhel6 || exit 1 ;
+   # blat $DB $IN -out=blast8 -prot $OUT || exit 1 ;
+   # module unload blat/rhel6 || exit 1 ;
+}
+
+# Function to execute AFTER running the BLAST, for each sub-task
+function AFTER_BLAST {
+   local IN=$1 # Query files
+   local OUT=$2 # Blast files
+   ### Filter by best-match:
+   # sort $OUT | perl $PDIR/../../Scripts/BlastTab.best_hit_sorted.pl > $OUT.bm
+   ### Filter by Bit-score 60:
+   # awk '$12>=60' $OUT > $OUT.bs60
+   ### Filter by corrected identity 95 (only if it has the additional 13th column):
+   # awk '$3*$4/$13 >= 95' $OUT > $OUT.ci95
+   ### Don't do anything:
+   true ;
+}
+
+# Function to execute ONLY ONCE at the end, to concatenate the results
+function END {
+   local PREFIX=$1 # Prefix of all Blast files
+   local OUT=$2 # Single Blast output (to be created).
+   ### Simply concatenate files:
+   # cat $PREFIX.*.blast > $OUT
+   ### Concatenate only the filtered files (if filtering in AFTER_BLAST):
+   # cat $PREFIX.*.blast.bs60 > $OUT
+   ### Sort the BLAST by query (might require considerable RAM):
+   # sort -k 1 $PREFIX.*.blast > $OUT
+   ### Don't do anyhthing:
+   true ;
+}
+

data/utils/enveomics/Pipelines/blast.pbs/FastA.split.pl

@@ -0,0 +1,55 @@
+#!/usr/bin/env perl
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update Oct-13-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Symbol;
+
+my ($file, $base, $outN) = @ARGV;
+
+$outN ||= 12;
+($file and $base) or die "
+Usage
+   $0 in_file.fa out_base[ no_files]
+   
+   in_file.fa	Input file in FastA format.
+   out_base	Prefix for the name of the output files.  It will
+   		be appended with .<i>.fa, where <i> is a consecutive
+		number starting in 1.
+   no_files	Number of files to generate.  By default: 12.
+
+";
+
+
+my @outSym = ();
+for my $i (1 .. $outN){
+   $outSym[$i-1] = gensym;
+   open $outSym[$i-1], ">", "$base.$i.fa" or
+      die "I can not create the file: $base.$i.fa: $!\n";
+}
+
+
+my($i, $seq) = (-1, '');
+open FILE, "<", $file or die "I can not read the file: $file: $!\n";
+while(my $ln=<FILE>){
+   next if $ln=~/^;/;
+   if($ln =~ m/^>/){
+      print { $outSym[$i % $outN] } $seq if $seq;
+      $i++;
+      $seq = '';
+   }
+   $seq.=$ln;
+}
+print { $outSym[$i % $outN] } $seq if $seq;
+close FILE;
+
+for(my $j=0; $j<$outN; $j++){
+   close $outSym[$j];
+}
+
+print STDERR "Sequences: ".($i+1)."\nFiles: $outN\n";
+

data/utils/enveomics/Pipelines/blast.pbs/README.md

@@ -0,0 +1,127 @@
+@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
+
+@update: Feb-20-2014
+
+@license: artistic 2.0
+
+@status: auto
+
+@pbs: yes
+
+# IMPORTANT
+
+This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
+are free to use it in other platforms with adequate adjustments.
+
+# PURPOSE
+
+Simplifies submitting and tracking large BLAST jobs in cluster.
+
+# HELP
+
+1. Files preparation:
+
+   1.1. Obtain the enveomics package in the cluster. You can use: `git clone https://github.com/lmrodriguezr/enveomics.git`
+   
+   1.2. Prepare the query sequences and the database.
+   
+   1.3. Copy the file `CONFIG.mock.bash` to `CONFIG.<name>.bash`, where `<name>` is a
+      short name for your project (avoid characters other than alphanumeric).
+   
+   1.4. Change the variables in `CONFIG.<name>.bash`. The **Queue and resources** and the
+      **Pipeline** variables are very standard, and can be kept unchanged. The **Paths**
+      variables indicate where your input files are and where the output files are to
+      be created, so check them carefully. Finally, the **FUNCTIONS** define the core
+      functionality of the pipeline, and should also be reviewed. By default, the
+      Pipeline simply runs BLAST+, with default parameters and tabular output with two
+      extra columns (qlen and slen). However, additional functionality can easily be
+      incorporated via these functions, such as BLAST filtering, concatenation, sorting,
+      or even execution of other programs instead of BLAST, such as BLAT, etc. Note that
+      the output MUST be BLAST-like tabular, because this is the only format supported
+      to check completeness and recover incomplete runs.
+   
+2. Pipeline execution:
+   
+   2.1. To initialize a run, execute: `./RUNME.bash <name> run`.
+
+   2.2. To check the status of a job, execute: `./RUNME.bash <name> check`.
+
+   2.3. To pause a run, execute: `./RUNME.bash <name> pause` (see 2.1 to resume).
+
+   2.4. To check if your CONFIG defines all required parameters, execute: `./RUNME.bash <name> dry`.
+
+   2.5. To review all the e/o files in the run, execute: `./RUNME.bash <name> eo`.
+
+3. Finalizing:
+   
+   3.1. `./RUNME.bash <name> check` will inform you if a project finished. If it finished successfully,
+      you can review your (split) results in $SCRATCH/results. If you concatenated the results in the
+      `END` function, you should have a file with all the results in $SCRATCH/<name>.blast.
+
+   3.2. Usually, checking the e/o files at the end is a good idea (`./RUNME.bash <name> eo`). However,
+      bear in mind that this Pipeline can overcome several errors and is robust to most failures, so
+      don't be alarmed at the first sight of errors.
+
+# Comments
+
+* Some scripts contained in this package are actually symlinks to files in the
+  _Scripts_ folder.  Check the existance of these files when copied to
+  the cluster.
+
+# Troubleshooting
+
+1. Do I really have to change directory (`cd`) to the pipeline's folder everytime I want to execute
+   something?
+   
+   No.  Not really.  For simplicity, this file tells you to execute `./RUNME.bash`.  However, you don't
+   really have to be there, you can execute it from any location.  For example, if you saved enveomics in
+   your home directory, you can just execute `~/enveomics/blast.pbs/RUNME.bash` insted from any location
+   in the head node.
+
+2. When I check a project, few sub-jobs are Active for much longer than the others. How do I know if those
+   sub-jobs are really active?
+   
+   Lets review an example of a problematic run. When you run `./RUNME.bash <name> check`, you see the
+   following in the "Active jobs" section:
+   ````
+   Idle: 155829.shared-sched.pace.gatech.edu: 02: 00: Mon Mar 17 14:10:28 EDT 2014
+     Sub-jobs:500 Active:4 ( 0.8% ) Eligible:0 ( 0.0% ) Blocked:0 ( 0.0% ) Completed:496 ( 99.2% ) 
+   Idle: 155830.shared-sched.pace.gatech.edu: 02: 00: Mon Mar 17 14:10:28 EDT 2014
+   
+   Running jobs: 0.
+   Idle jobs: 2.
+   ````
+   That means that the job "155829.shared-sched.pace.gatech.edu" has four Active jobs, while all the others are Completed. This is
+   a sign of something problematic.  You can see the complete status of each array using
+   `checkjob -v <JOB_NAME>`. In our example above, you would run `checkjob -v 155829`. In the output
+   of checkjob, most jobs should report "Completed". In this example, there are four jobs that are not
+   complete:
+   ````
+   ...
+   387 : 155829[387] : Completed
+   388 : 155829[388] : Running
+   389 : 155829[389] : Running
+   390 : 155829[390] : Running
+   391 : 155829[391] : Running
+   392 : 155829[392] : Completed
+   ...
+   ````
+   So you can simply check these sub-jobs in more detail. For example, if I run `checkjob -v 155829[388]`,
+   I see that the job is running in the machine `iw-k30-12.pace.gatech.edu` (Task Distribution), so I can try
+   to login to that machine to check if the job is actually running, using `top -u $(whoami)`. However, when
+   I run `ssh iw-k30-12`, I got a "Connection closed" error, which means that the machine hung up. At this point,
+   you might want to try one of the following solutions:
+
+   2.1. Pause the project using `./RUNME.bash <name> pause`, wait a few minutes, and resume using
+        `./RUNME.bash <name> run`. If you tried this a couple of times and you still have sub-jobs hanging, try:
+   
+   2.2. Check if your sub-jobs finished. Sometimes sub-jobs die too soon to return a success code, but they actually
+        finished. Just run the following command: `ls <SCRATCH>/<name>/success/02.* | wc -l`, where `<SCRATCH>` is the
+	value you set for the `SCRATCH` variable in the CONFIG file, and `<name>` is the name of your project. If the
+	output of that command is a number, and that number is exactly six times the number of jobs (`MAX_JOBS` in the
+	CONFIG file, typically 500), then your step 2 actually finished. In my case, I have 500 jobs, and the output
+	was 3000, so my job finished successfully, but the pipeline didn't notice. You can manually tell the pipeline
+	to go on running: `touch <SCRATCH>/<name>/success/02`, and pausing/resuming the project (see 2.1 above). If
+	the output is not the expected number (in my case, 3000, which is 6*500), DON'T RUN `touch`, just try the
+	solution 2.1 above once again.
+

data/utils/enveomics/Pipelines/blast.pbs/RUNME.bash

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+##################### VARIABLES
+# Find the directory of the pipeline
+if [[ "$PDIR" == "" ]] ; then PDIR=$(dirname $(readlink -f $0)); fi ;
+CWD=$(pwd)
+
+# Load config
+if [[ "$PROJ" == "" ]] ; then PROJ="$1" ; fi
+if [[ "$TASK" == "" ]] ; then TASK="$2" ; fi
+if [[ "$TASK" == "" ]] ; then TASK="check" ; fi
+NAMES=$(ls $PDIR/CONFIG.*.bash | sed -e 's/.*CONFIG\./    o /' | sed -e 's/\.bash//');
+if [[ "$PROJ" == "" ]] ; then
+   if [[ "$HELP" == "" ]] ; then
+      echo "
+Usage:
+   $0 name task
+   
+   name	The name of the run.  CONFIG.name.bash must exist.
+   task	The action to perform.  One of:
+	o run: Executes the BLAST.
+	o check: Indicates the progress of the task (default).
+	o pause: Cancels running jobs (resume using run).
+	o dry: Checks that the parameters are correct, but doesn't run.
+	o eo: Review all eo files produced in the project.
+
+   See $PDIR/README.md for more information.
+   
+   Available names are:
+$NAMES
+" >&2
+   else
+      echo "$HELP   
+   Available names are:
+$NAMES
+" >&2
+   fi
+   exit 1
+fi
+if [[ ! -e "$PDIR/CONFIG.$PROJ.bash" ]] ; then
+   echo "$0: Error: Impossible to find $PDIR/CONFIG.$PROJ.bash, available names are:
+$NAMES" >&2
+   exit 1
+fi
+source "$PDIR/CONFIG.$PROJ.bash" ;
+SCRATCH="$SCRATCH_DIR/$PROJ" ;
+MINVARS="PDIR=$PDIR,SCRATCH=$SCRATCH,PROJ=$PROJ" ;
+case $QUEUE in
+bioforce-6)
+   MAX_H=120 ;;
+iw-shared-6)
+   MAX_H=12 ;;
+micro-largedata)
+   MAX_H=120 ;;
+biocluster-6 | biohimem-6 | microcluster)
+   MAX_H=240 ;;
+*)
+   echo "Unrecognized queue: $QUEUE." >&2 ;
+   exit 1 ;;
+esac ;
+
+##################### FUNCTIONS
+function REGISTER_JOB {
+   local STEP=$1
+   local SUBSTEP=$2
+   local MESSAGE=$3
+   local JOBID=$4
+
+   if [[ "$JOBID" != "" ]] ; then
+      MESSAGE="$MESSAGE [$JOBID]" ;
+      echo "$STEP: $SUBSTEP: $(date)" >> "$SCRATCH/log/active/$JOBID" ;
+   fi
+   echo "$MESSAGE." >> "$SCRATCH/log/status/$STEP" ;
+}
+
+function LAUNCH_JOB {
+   local STEP=$1
+   local SUBSTEP=$2
+   local MESSAGE=$3
+   local BASHFILE=$4
+   
+   cd "$SCRATCH/log/eo" ;
+   date >> "$SCRATCH/etc/trials" ;
+   source "$BASHFILE" || exit 1 ;
+   cd $CWD ;
+   if [[ "$SENTINEL_JOBID" != "" ]] ; then
+      REGISTER_JOB "$STEP" "$SUBSTEP" "Guarding job $NEW_JOBID" "$SENTINEL_JOBID" ;
+   fi ;
+   REGISTER_JOB "$STEP" "$SUBSTEP" "$MESSAGE" "$NEW_JOBID" ;
+   echo $NEW_JOBID ;
+}
+
+function JOB_DONE {
+   STEP=$1
+
+   echo "Done." >> "$SCRATCH/log/status/$STEP" ;
+   touch "$SCRATCH/success/$STEP" ;
+   echo -n '# ' > "$SCRATCH/etc/trials" ;
+}
+
+##################### RUN
+# Execute task
+if [[ ! -e "$PDIR/TASK.$TASK.bash" ]] ; then
+   echo "Unrecognized task: $TASK." >&2 ;
+   exit 1 ;
+else
+   source "$PDIR/TASK.$TASK.bash"
+fi
+