miga-base 0.7.23.0 → 0.7.25.3

data/utils/enveomics/Pipelines/blast.pbs/RUNME.bash

@@ -1,109 +0,0 @@
-#!/bin/bash
-
-##################### VARIABLES
-# Find the directory of the pipeline
-if [[ "$PDIR" == "" ]] ; then PDIR=$(dirname $(readlink -f $0)); fi ;
-CWD=$(pwd)
-
-# Load config
-if [[ "$PROJ" == "" ]] ; then PROJ="$1" ; fi
-if [[ "$TASK" == "" ]] ; then TASK="$2" ; fi
-if [[ "$TASK" == "" ]] ; then TASK="check" ; fi
-NAMES=$(ls $PDIR/CONFIG.*.bash | sed -e 's/.*CONFIG\./    o /' | sed -e 's/\.bash//');
-if [[ "$PROJ" == "" ]] ; then
-   if [[ "$HELP" == "" ]] ; then
-      echo "
-Usage:
-   $0 name task
-   
-   name	The name of the run.  CONFIG.name.bash must exist.
-   task	The action to perform.  One of:
-	o run: Executes the BLAST.
-	o check: Indicates the progress of the task (default).
-	o pause: Cancels running jobs (resume using run).
-	o dry: Checks that the parameters are correct, but doesn't run.
-	o eo: Review all eo files produced in the project.
-
-   See $PDIR/README.md for more information.
-   
-   Available names are:
-$NAMES
-" >&2
-   else
-      echo "$HELP   
-   Available names are:
-$NAMES
-" >&2
-   fi
-   exit 1
-fi
-if [[ ! -e "$PDIR/CONFIG.$PROJ.bash" ]] ; then
-   echo "$0: Error: Impossible to find $PDIR/CONFIG.$PROJ.bash, available names are:
-$NAMES" >&2
-   exit 1
-fi
-source "$PDIR/CONFIG.$PROJ.bash" ;
-SCRATCH="$SCRATCH_DIR/$PROJ" ;
-MINVARS="PDIR=$PDIR,SCRATCH=$SCRATCH,PROJ=$PROJ" ;
-case $QUEUE in
-bioforce-6)
-   MAX_H=120 ;;
-iw-shared-6)
-   MAX_H=12 ;;
-micro-largedata)
-   MAX_H=120 ;;
-biocluster-6 | biohimem-6 | microcluster)
-   MAX_H=240 ;;
-*)
-   echo "Unrecognized queue: $QUEUE." >&2 ;
-   exit 1 ;;
-esac ;
-
-##################### FUNCTIONS
-function REGISTER_JOB {
-   local STEP=$1
-   local SUBSTEP=$2
-   local MESSAGE=$3
-   local JOBID=$4
-
-   if [[ "$JOBID" != "" ]] ; then
-      MESSAGE="$MESSAGE [$JOBID]" ;
-      echo "$STEP: $SUBSTEP: $(date)" >> "$SCRATCH/log/active/$JOBID" ;
-   fi
-   echo "$MESSAGE." >> "$SCRATCH/log/status/$STEP" ;
-}
-
-function LAUNCH_JOB {
-   local STEP=$1
-   local SUBSTEP=$2
-   local MESSAGE=$3
-   local BASHFILE=$4
-   
-   cd "$SCRATCH/log/eo" ;
-   date >> "$SCRATCH/etc/trials" ;
-   source "$BASHFILE" || exit 1 ;
-   cd $CWD ;
-   if [[ "$SENTINEL_JOBID" != "" ]] ; then
-      REGISTER_JOB "$STEP" "$SUBSTEP" "Guarding job $NEW_JOBID" "$SENTINEL_JOBID" ;
-   fi ;
-   REGISTER_JOB "$STEP" "$SUBSTEP" "$MESSAGE" "$NEW_JOBID" ;
-   echo $NEW_JOBID ;
-}
-
-function JOB_DONE {
-   STEP=$1
-
-   echo "Done." >> "$SCRATCH/log/status/$STEP" ;
-   touch "$SCRATCH/success/$STEP" ;
-   echo -n '# ' > "$SCRATCH/etc/trials" ;
-}
-
-##################### RUN
-# Execute task
-if [[ ! -e "$PDIR/TASK.$TASK.bash" ]] ; then
-   echo "Unrecognized task: $TASK." >&2 ;
-   exit 1 ;
-else
-   source "$PDIR/TASK.$TASK.bash"
-fi
-

data/utils/enveomics/Pipelines/blast.pbs/TASK.check.bash

@@ -1,128 +0,0 @@
-#!/bin/bash
-
-##################### RUN
-# Check if it was sourced from RUNME.bash
-if [[ "$PDIR" == "" ]] ; then
-   echo "$0: Error: This file is not stand-alone." >&2
-   echo "  Execute RUNME.bash as described in the README.txt file" >&2
-   exit 1
-fi
-
-# Check if the project exists
-if [[ ! -d "$SCRATCH" ]] ; then
-   echo "The project $PROJ doesn't exist at $SCRATCH_DIR." >&2
-   echo "  Execute '$PDIR/RUNME.bash $PROJ run' first." >&2
-   exit 1
-fi
-
-# Get log:
-echo "==[ Running tasks ]=="
-for i in $(ls $SCRATCH/log/status/* 2>/dev/null) ; do
-   echo "  $(basename $i): $(tail -n 1 $i)"
-done
-echo ""
-
-# Get active jobs:
-echo "==[ Active jobs ]=="
-job_r=0
-job_i=0
-job_c=0
-for i in $(ls $SCRATCH/log/active/* 2>/dev/null) ; do
-   jid=$(basename $i)
-   stat=$(qstat -f1 $jid 2>&1)
-   state=$(echo "$stat" | grep '^ *job_state = ' | sed -e 's/.*job_state = //')
-   case $state in
-   C)
-      code=$(echo "$stat" | grep '^ *exit_status = ' | sed -e 's/.*exit_status = //')
-      if [[ "$code" == "0" ]] ; then
-         mv "$i" "$SCRATCH/log/done/"
-	 let job_c=$job_c+1
-      else
-	 echo "Warning: Job $jid ($(cat $i|tr -d '\n')) failed with code $code." >&2
-	 echo "  see errors at: $(echo "$stat" | grep '^ *Error_Path = ' | sed -e 's/.*Error_Path = //')"
-         mv "$i" "$SCRATCH/log/failed/"
-      fi ;;
-   R)
-      echo "  Running: $jid: $(cat "$i")"
-      let job_r=$job_r+1 ;;
-   [HQW])
-      echo "  Idle: $jid: $(cat "$i")"
-      let job_i=$job_i+1 ;;
-   E)
-      echo "  Canceling: $jid: $(cat "$i")" ;;
-   *)
-      tmp_err=$(echo "$stat" | grep ERROR)
-      if [[ "$tmp_err" == "" ]] ; then
-	 echo "Warning: Unrecognized state: $jid: $state." >&2
-	 echo "  Please report this problem." >&2
-      else
-	 echo "  Error: $jid: $tmp_err"
-      fi ;;
-   esac
-   #subjobs=$(echo "$stat" | grep 'Sub-jobs:' | sed -e 's/.*: *//')
-   #if [[ "$subjobs" -gt 0 ]] ; then
-   #   echo "$stat" | grep '^ *\(Sub-jobs\|Active\|Eligible\|Blocked\|Completed\):' | sed -e 's/^ *//' | sed -e 's/  *//' | tr '\n' ' ' | sed -e 's/^/    /'
-   #   echo
-   #fi
-done
-if [[ $job_c -gt 0 ]] ; then
-   echo ""
-   echo "  Completed since last check: $job_c."
-fi
-if [[ $job_r -gt 0 || $job_i -gt 0 ]] ; then
-   echo ""
-   echo "  Running jobs: $job_r."
-   echo "  Idle jobs: $job_i."
-fi
-echo ""
-
-# Auto-trials
-echo "==[ Auto-trials ]=="
-if [[ -e "$SCRATCH/etc/trials" ]] ; then
-   trials=$(cat "$SCRATCH/etc/trials" | wc -l | sed -e 's/ //g')
-   if [[ $trials -gt 1 ]] ; then
-      echo "  $trials trials attempted:"
-   else
-      echo "  No recent failures in the current step, job launched:"
-   fi
-   cat "$SCRATCH/etc/trials" | sed -e 's/^/  o /' | sed -e 's/# $/No active trials\n/g'
-fi
-echo ""
-
-# Step-specific checks:
-echo "==[ Step summary ]=="
-todo=1
-if [[ -e "$SCRATCH/success/00" ]] ; then
-   echo "  Successful project initialization."
-   if [[ -e "$SCRATCH/success/01" ]] ; then
-      echo "  Successful input preparation."
-      if [[ -e "$SCRATCH/success/02" ]] ; then
-	 echo "  Successful BLAST execution."
-	 if [[ -e "$SCRATCH/success/02" ]] ; then
-	    echo "  Successful concatenation."
-	    echo "  Project finished successfully!"
-	    todo=0
-	 else
-	    echo "  Concatenating results."
-	 fi
-      else
-	 echo "  Running BLAST."
-      fi
-   else
-      echo "  Preparing input."
-   fi
-else
-   echo "  Initializing project."
-fi
-
-if [[ "$todo" -eq 1 && $job_r -eq 0 && $job_i -eq 0 ]] ; then
-   echo "  Job currently paused. To resume, execute:"
-   echo "  $PDIR/RUNME.bash $PROJ run"
-fi
-echo
-
-# Entire log
-echo "==[ Complete log ]=="
-for i in $(ls $SCRATCH/log/status/* 2>/dev/null) ; do
-   cat "$i" | sed -e "s/^/  $(basename $i): /"
-done

data/utils/enveomics/Pipelines/blast.pbs/TASK.dry.bash

@@ -1,16 +0,0 @@
-
-[[ "$QUEUE" == "" ]] && echo "Undefined QUEUE" >&2 && exit 1;
-[[ "$MAX_JOBS" == "" ]] && echo "Undefined MAX_JOBS" >&2 && exit 1;
-[[ "$PPN" == "" ]] && echo "Undefined PPN" >&2 && exit 1;
-[[ "$RAM" == "" ]] && echo "Undefined RAM" >&2 && exit 1;
-[[ "$SCRATCH_DIR" == "" ]] && echo "Undefined SCRATCH_DIR" >&2 && exit 1;
-[[ "$INPUT" == "" ]] && echo "Undefined INPUT" >&2 && exit 1;
-[[ "$DB" == "" ]] && echo "Undefined DB" >&2 && exit 1;
-[[ "$PROGRAM" == "" ]] && echo "Undefined PROGRAM" >&2 && exit 1;
-[[ "$MAX_TRIALS" == "" ]] && echo "Undefined MAX_TRIALS" >&2 && exit 1;
-[[ "$(type -t BEGIN)" == "function" ]] || ( echo "Undefined function BEGIN" && exit 1 ) ;
-[[ "$(type -t BEFORE_BLAST)" == "function" ]] || ( echo "Undefined function BEFORE_BLAST" && exit 1 ) ;
-[[ "$(type -t RUN_BLAST)" == "function" ]] || ( echo "Undefined function RUN_BLAST" && exit 1 ) ;
-[[ "$(type -t AFTER_BLAST)" == "function" ]] || ( echo "Undefined function AFTER_BLAST" && exit 1 ) ;
-[[ "$(type -t END)" == "function" ]] || ( echo "Undefined function END" && exit 1 ) ;
-

data/utils/enveomics/Pipelines/blast.pbs/TASK.eo.bash

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-##################### RUN
-# Check if it was sourced from RUNME.bash
-if [[ "$PDIR" == "" ]] ; then
-   echo "$0: Error: This file is not stand-alone." >&2
-   echo "  Execute RUNME.bash as described in the README.txt file" >&2 ;
-   exit 1 ;
-fi ;
-
-# Check if the project exists
-if [[ ! -d "$SCRATCH" ]] ; then
-   echo "The project $PROJ doesn't exist at $SCRATCH_DIR." >&2 ;
-   echo "  Execute '$PDIR/RUNME.bash $PROJ run' first." >&2 ;
-   exit 1 ;
-fi ;
-
-# Review errors
-(echo -e "==[ Last 10 lines of all e files ]==\nPress q to exit\n" ; tail -n 10 $SCRATCH/log/eo/*.e* ) | less
-# Review output
-(echo -e "==[ Last 100 lines of all o files ]==\nPress q to exit\n" ; tail -n 100 $SCRATCH/log/eo/*.o* ) | less
-

data/utils/enveomics/Pipelines/blast.pbs/TASK.pause.bash

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-##################### RUN
-# Check if it was sourced from RUNME.bash
-if [[ "$PDIR" == "" ]] ; then
-   echo "$0: Error: This file is not stand-alone." >&2
-   echo "  Execute RUNME.bash as described in the README.txt file" >&2 ;
-   exit 1 ;
-fi ;
-
-# Get active jobs:
-echo "======[ check ]======"
-job_r=0;
-job_i=0;
-job_c=0;
-
-echo "======[ pause ]======"
-for i in $(ls $SCRATCH/log/active/* 2>/dev/null) ; do
-   echo "  Pausing $jid." ;
-   jid=$(basename $i) ;
-   qdel $jid ;
-done ;
-
-# Restart auto-trials
-echo -n > "$SCRATCH/etc/trials" ;
-

data/utils/enveomics/Pipelines/blast.pbs/TASK.run.bash

@@ -1,89 +0,0 @@
-#!/bin/bash
-
-##################### RUN
-# Check if it was sourced from RUNME.bash
-if [[ "$PDIR" == "" ]] ; then
-   echo "$0: Error: This file is not stand-alone.  Execute RUNME.bash as described in the README.txt file" >&2
-   exit 1
-fi
-
-# Check if too many auto-trials were attempted
-trials=0 ;
-if [[ -e "$SCRATCH/etc/trials" ]] ; then
-   trials=$(cat "$SCRATCH/etc/trials" | wc -l | sed -e 's/ //g');
-   if [[ $trials -ge $MAX_TRIALS ]] ; then
-      echo "The maximum number of trials was already attempted, halting." >&2 ;
-      exit 1 ;
-   fi ;
-fi ;
-
-# Create the scratch directory
-if [[ ! -d "$SCRATCH" ]] ; then mkdir -p "$SCRATCH" || exit 1 ; fi;
-
-if [[ ! -e "$SCRATCH/success/00" ]] ; then
-   # 00. Initialize the project
-   echo "00. Initializing project." >&2 ;
-   mkdir -p "$SCRATCH/tmp" "$SCRATCH/etc" "$SCRATCH/results" "$SCRATCH/success" || exit 1 ;
-   mkdir -p "$SCRATCH/log/active" "$SCRATCH/log/done" "$SCRATCH/log/failed" || exit 1 ;
-   mkdir -p "$SCRATCH/log/status" "$SCRATCH/log/eo" || exit 1 ;
-   echo "Preparing structure." >> "$SCRATCH/log/status/00" ;
-   # Build 01.bash
-   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM' -v '$MINVARS' -N '$PROJ-01' \\
-      '$PDIR/01.pbs.bash'|tr -d '\\n')" \
-      > "$SCRATCH/etc/01.bash" || exit 1 ;
-   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afterany:\$NEW_JOBID\" \\
-      -v \"$MINVARS,STEP=01,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-01-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
-      >> "$SCRATCH/etc/01.bash" || exit 1 ;
-   # Build 02.bash
-   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM,nodes=1:ppn=$PPN' \\
-      -v '$MINVARS' -N '$PROJ-02' -t '1-$MAX_JOBS' '$PDIR/02.pbs.bash'|tr -d '\\n')" \
-      > "$SCRATCH/etc/02.bash" \
-      || exit 1 ;
-   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afteranyarray:\$NEW_JOBID\" \\
-      -v \"$MINVARS,STEP=02,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-02-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
-      >> "$SCRATCH/etc/02.bash" \
-      || exit 1 ;
-   # Build 03.bash
-   echo "NEW_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=$MAX_H:00:00,mem=$RAM' -v '$MINVARS' -N '$PROJ-03' \\
-      '$PDIR/03.pbs.bash'|tr -d '\\n')" \
-      > "$SCRATCH/etc/03.bash" || exit 1 ;
-   echo "SENTINEL_JOBID=\$(qsub -q '$QUEUE' -l 'walltime=2:00:00' -W \"depend=afterany:\$NEW_JOBID\" \\
-      -v \"$MINVARS,STEP=03,AFTERJOB=\$NEW_JOBID\" -N '$PROJ-03-sentinel' '$PDIR/sentinel.pbs.bash'|tr -d '\\n')" \
-      >> "$SCRATCH/etc/03.bash" || exit 1 ;
-   
-   JOB_DONE "00" ;
-fi ;
-
-if [[ ! -e "$SCRATCH/success/01" ]] ; then
-   # 01. Preparing input
-   echo "01. Preparing input." >&2 ;
-   JOB01=$(LAUNCH_JOB "01" "00" "Preparing input files" "$SCRATCH/etc/01.bash") ;
-   echo "  New job: $JOB01." >&2 ;
-else
-   if [[ ! -e "$SCRATCH/success/02" ]] ; then
-      # 02. Launching BLAST
-      echo "02. Launching BLAST." >&2 ;
-      JOB02=$(LAUNCH_JOB "02" "00" "Running BLAST" "$SCRATCH/etc/02.bash") ;
-      echo "  New job: $JOB02." >&2 ;
-      # Clean on resubmission
-      cleaned=0
-      echo -n "  Cleaning completed sub-jobs on $JOB02: " >&2 ;
-      for jobi in $(seq 1 $MAX_JOBS) ; do
-	 if [[ -e "$SCRATCH/success/02.$jobi" ]] ; then
-	    qdel "$JOB02""[$jobi]" &> /dev/null ;
-	    let cleaned=$cleaned+1 ;
-	 fi ;
-      done ;
-      echo "$cleaned sub-jobs completed." >&2 ;
-   else
-      if [[ ! -e "$SCRATCH/success/03" ]] ; then
-	 # 03. Finalize
-	 echo "03. Finalizing." >&2 ;
-	 JOB03=$(LAUNCH_JOB "03" "00" "Concatenating results" "$SCRATCH/etc/03.bash") ;
-	 echo "  New job: $JOB03." >&2 ;
-      else
-         echo "Project complete, nothing to run." ;
-      fi ;
-   fi ;
-fi ;
-

data/utils/enveomics/Pipelines/blast.pbs/sentinel.pbs.bash

@@ -1,29 +0,0 @@
-# blast.pbs pipeline
-# Sentinel script
-
-echo "Sentinel script after $AFTERJOB" ;
-
-# Step-specific checks
-if [[ "$STEP" == "02" ]] ; then
-   # Read configuration
-   cd $SCRATCH ;
-   TASK="dry" ;
-   source "$PDIR/RUNME.bash" ;
-   
-   # Check tasks
-   INCOMPLETE=0;
-   for i in $(seq 1 $MAX_JOBS) ; do
-      if [[ ! -e "$SCRATCH/success/02.$i" ]] ; then
-	 let INCOMPLETE=$INCOMPLETE+1 ;
-      fi ;
-   done
-   if [[ $INCOMPLETE -eq 0 ]] ; then
-      JOB_DONE "02" ;
-   else
-      echo "$INCOMPLETE incomplete jobs, re-launching step 02." ;
-   fi ;
-fi
-
-# Continue the workflow
-"$PDIR/RUNME.bash" "$PROJ" run || exit 1 ;
-

data/utils/enveomics/Pipelines/idba.pbs/README.md

@@ -1,49 +0,0 @@
-@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
-
-@update: Feb-26-2015
-
-@license: artistic 2.0
-
-@status: auto
-
-@pbs: yes
-
-# IMPORTANT
-
-This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
-are free to use it in other platforms with adequate adjustments.
-
-# PURPOSE
-
-Performs assembly using IDBA-UD, designed for Single-Cell Genomics and Metagenomics.
-
-# HELP
-
-1. Files preparation:
-
-   1.1. Obtain the enveomics package in the cluster. You can use:
-      `git clone https://github.com/lmrodriguezr/enveomics.git`
-   
-   1.2. Prepare the trimmed reads (e.g., use trim.bs) in interposed FastA format. Files
-      must be raw, not zipped or packaged. Filenames must conform the format:
-      <name>.CoupledReads.fa, where <name> is the name of the sample. Locate all the
-      files within a folder named 04.trimmed_fasta, within your project folder. If you
-      used trim.pbs, no further action is necessary.
-   
-2. Pipeline execution:
-   
-   2.1. Simply execute `./RUNME.bash <dir> <data_type>`, where `<dir>` is the folder containing
-      the 04.trimmed_fasta folder, and `<data_type>` is a supported type of data (see help
-      message running `./RUNME.bash` without arguments).
-
-3. What to expect:
-
-   By the end of the run, you should find the folder *05.assembly*, including the following
-   files for each dataset:
-   
-   3.1. `<dataset>`: The IDBA output folder.
-   
-   3.2. `<dataset>.AllContigs.fna`: All contigs longer than 200bp in FastA format.
-
-   3.2. `<dataset>.LargeContigs.fna`: Contigs longer than 500bp in FastA format.
-