miga-base 0.2.0.6

data/scripts/mytaxa.bash

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+DIR="$PROJECT/data/07.annotation/02.taxonomy/01.mytaxa"
+[[ -d "$DIR" ]] || mkdir -p "$DIR"
+cd "$DIR"
+
+# Initialize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.start"
+MT=$(dirname -- $(which MyTaxa))
+
+# Check type of dataset
+MULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --multi \
+   | wc -l | awk '{print $1}')
+if [[ "$MULTI" -eq "1" ]] ; then
+   # Check requirements
+   if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
+      echo "Cannot locate the database: $MT/AllGenomes.faa.dmnd:" \
+	 "no such file or directory" >&2
+      exit 1
+   fi
+   if [[ ! -d "$MT/db" ]] ; then
+      echo "Cannot locate the MyTaxa index: $MT/db:" \
+	 "no such file or directory" >&2
+      exit 1
+   fi
+   if [[ ! -d "$MT/utils" ]] ; then
+      echo "Cannot locate the MyTaxa utilities: $MT/utils:" \
+	 "no such file or directory" >&2
+      exit 1
+   fi
+   
+   # Execute search
+   diamond blastp -q "../../../06.cds/$DATASET.faa" -d "$MT/AllGenomes.faa" \
+      -a "$DATASET.daa" -k 5 -p "$CORES" --min-score 60
+   diamond view -a "$DATASET.daa" -o "$DATASET.blast"
+
+   # Prepare MyTaxa input, execute MyTaxa, and generate profiles
+   [[ -e "../../../06.cds/$DATASET.gff2.gz" ]] \
+      && [[ ! -e "../../../06.cds/$DATASET.gff2" ]] \
+      && gunzip "../../../06.cds/$DATASET.gff2.gz"
+   [[ -e "../../../06.cds/$DATASET.gff3.gz" ]] \
+      && [[ ! -e "../../../06.cds/$DATASET.gff3" ]] \
+      && gunzip "../../../06.cds/$DATASET.gff3.gz"
+   if [[ -e "../../../06.cds/$DATASET.gff2" ]] ; then
+      # GFF2
+      perl "$MT/utils/infile_convert.pl" -f gff2 \
+	 "../../../06.cds/$DATASET.gff2" "$DATASET.blast" \
+	 | sort -k 13 > "$DATASET.mytaxain"
+      "$MT/MyTaxa" "$DATASET.mytaxain" "$DATASET.mytaxa" "0.5"
+      perl "$MT/utils/MyTaxa.distribution.pl" -m "$DATASET.mytaxa" \
+	 -g "../../../06.cds/$DATASET.gff2" -f gff2 \
+	 -I "$DATASET.mytaxa.innominate" -G "$DATASET.mytaxa.genes" \
+	 -K "$DATASET.mytaxa.krona" -u
+   elif [[ -e "../../../06.cds/$DATASET.gff3" ]] ; then
+      # GFF3
+      perl "$MT/utils/infile_convert.pl" -f gff3 \
+	 "../../../06.cds/$DATASET.gff3" "$DATASET.blast" | sort -k 13 \
+	 > "$DATASET.mytaxain"
+      "$MT/MyTaxa" "$DATASET.mytaxain" "$DATASET.mytaxa" "0.5"
+      perl "$MT/utils/MyTaxa.distribution.pl" -m "$DATASET.mytaxa" \
+	 -g "../../../06.cds/$DATASET.gff3" -f gff3 \
+	 -I "$DATASET.mytaxa.innominate" -G "$DATASET.mytaxa.genes" \
+	 -K "$DATASET.mytaxa.krona" -u
+   else
+      # No GFF
+      perl "$MT/utils/infile_convert.pl" -f no "LOREM_IPSUM" "$DATASET.blast" \
+	 | sort -k 13 > "$DATASET.mytaxain"
+      "$MT/MyTaxa" "$DATASET.mytaxain" "$DATASET.mytaxa" "0.5"
+      perl "$MT/utils/MyTaxa.distribution.pl" -m "$DATASET.mytaxa" \
+	 -I "$DATASET.mytaxa.innominate" -G "$DATASET.mytaxa.genes" \
+	 -K "$DATASET.mytaxa.krona" -u
+   fi
+
+   # Execute Krona
+   ktImportText -o "$DATASET.html" -n biota "$DATASET.mytaxa.krona,$DATASET"
+
+   # Gzip and cleanup
+   [[ -e "../../../06.cds/$DATASET.gff2" ]] \
+      && gzip -9 -f "../../../06.cds/$DATASET.gff2"
+   [[ -e "../../../06.cds/$DATASET.gff3" ]] \
+      && gzip -9 -f "../../../06.cds/$DATASET.gff3"
+   gzip -9 -f "$DATASET.mytaxain"
+   gzip -9 -f "$DATASET.blast"
+   rm "$DATASET.daa"
+fi
+
+# Finalize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.done"
+miga add_result -P "$PROJECT" -D "$DATASET" -r mytaxa
+

data/scripts/mytaxa_scan.bash

@@ -0,0 +1,85 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+DIR="$PROJECT/data/07.annotation/03.qa/02.mytaxa_scan"
+[[ -d "$DIR" ]] || mkdir -p "$DIR"
+cd "$DIR"
+
+# Initialize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.start"
+MT=$(dirname -- $(which MyTaxa))
+TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
+trap "rm -rf $TMPDIR; exit" SIGHUP SIGINT SIGTERM
+
+# Check type of dataset
+NOMULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --no-multi \
+   | wc -l | awk '{print $1}')
+if [[ "$NOMULTI" -eq "1" ]] ; then
+   # Check requirements
+   if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
+      echo "Cannot locate the database: $MT/AllGenomes.faa.dmnd:" \
+	 "no such file or directory" >&2
+      exit 1
+   fi
+   if [[ ! -d "$MT/db" ]] ; then
+      echo "Cannot locate the MyTaxa index: $MT/db:" \
+	 "no such file or directory" >&2
+      exit 1
+   fi
+   if [[ ! -d "$MT/utils" ]] ; then
+      echo "Cannot locate the MyTaxa utilities: $MT/utils:" \
+	 "no such file or directory" >&2
+      exit 1
+   fi
+   
+   if [[ ! -s "$DATASET.mytaxa" ]] ; then
+      # Execute search
+      if [[ ! -s "$DATASET.blast" ]] ; then
+	 diamond blastp -q "../../../06.cds/$DATASET.faa" \
+	    -d "$MT/AllGenomes.faa" -k 5 -p "$CORES" --min-score 60 \
+	    -a "$DATASET.daa" -t "$TMPDIR"
+	 diamond view -a "$DATASET.daa" -o "$DATASET.blast" -t "$TMPDIR"
+      fi
+
+      # Prepare MyTaxa input, execute MyTaxa, and generate profiles
+      perl "$MT/utils/infile_convert.pl" -f no "LOREM_IPSUM" "$DATASET.blast" \
+	 | sort -k 13 > "$DATASET.mytaxain"
+      "$MT/MyTaxa" "$DATASET.mytaxain" "$DATASET.mytaxa" "0.5"
+   fi
+   ruby "$MIGA/utils/mytaxa_scan.rb" "../../../06.cds/$DATASET.faa" \
+      "$DATASET.mytaxa" "$DATASET.wintax"
+   echo "
+   source('$MIGA/utils/mytaxa_scan.R');
+   pdf('$DATASET.pdf', 12, 7);
+   mytaxa.scan('$DATASET.wintax');
+   dev.off();
+   " | R --vanilla
+
+   # Extract genes from flagged regions
+   [[ -d "$DATASET.reg" ]] || mkdir "$DATASET.reg"
+   if [[ -e "$DATASET.wintax.regions" ]] ; then
+      i=0
+      for win in $(cat "$DATASET.wintax.regions") ; do
+	 let i=$i+1
+	 awk "NR==$win" "$DATASET.wintax.genes" | tr "\\t" "\\n" \
+	    > "$DATASET.reg/$i.ids"
+	 FastA.filter.pl -q "$DATASET.reg/$i.ids" \
+	    "../../../06.cds/$DATASET.faa" > "$DATASET.reg/$i.faa"
+      done
+   fi
+
+   # Clean
+   [[ -e "$DATASET.daa" ]] && rm "$DATASET.daa"
+   [[ -s "$DATASET.blast" && ! -s "$DATASET.blast.gz" ]] \
+      && gzip -9 -f "$DATASET.blast"
+   [[ -s "$DATASET.mytaxain" && ! -s "$DATASET.mytaxain.gz" ]] \
+      && gzip -9 -f "$DATASET.mytaxain"
+fi
+
+# Finalize
+rm -R "$TMPDIR"
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.done"
+miga add_result -P "$PROJECT" -D "$DATASET" -r mytaxa_scan
+

data/scripts/ogs.bash

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+cd "$PROJECT/data/10.clades/03.ogs"
+
+# Initialize
+date "+%Y-%m-%d %H:%M:%S %z" > "miga-project.start"
+
+echo -n "" > miga-project.log
+DS=$(miga list_datasets -P "$PROJECT" --ref --no-multi)
+
+# Extract RBMs
+[[ -d miga-project.rbm ]] || mkdir miga-project.rbm
+for i in $DS ; do
+   for j in $DS ; do
+      file="miga-project.rbm/$i-$j.rbm"
+      [[ -s $file ]] && continue
+      echo "SELECT id1,id2,id,0,0,0,0,0,0,0,evalue,bitscore from rbm" \
+	 "where seq1='$i' and seq2='$j' ;" \
+	 | sqlite3 "../../09.distances/02.aai/$i.db" | tr "\\|" "\\t" \
+	 > $file
+      [[ -s $file ]] || rm $file
+   done
+   echo "$i" >> miga-project.log
+done
+
+# Estimate OGs
+ogs.mcl.rb -o miga-project.ogs -d miga-project.rbm -t $CORES
+ogs.stats.rb -o miga-project.ogs -j miga-project.stats
+
+# Finalize
+date "+%Y-%m-%d %H:%M:%S %z" > "miga-project.done"
+miga add_result -P "$PROJECT" -r ogs
+

data/scripts/read_quality.bash

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+cd "$PROJECT/data/03.read_quality"
+
+b=$DATASET
+
+# Initialize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.start"
+
+# FastQC
+[[ -d "$b.fastqc" ]] || mkdir "$b.fastqc"
+fastqc ../02.trimmed_reads/$b.[12].clipped.fastq -o $b.fastqc
+
+# SolexaQA++
+[[ -d "$b.solexaqa" ]] || mkdir "$b.solexaqa"
+exists ../02.trimmed_reads/$b.[12].*.pdf \
+   && mv ../02.trimmed_reads/$b.[12].*.pdf "$b.solexaqa/"
+
+# Clean 02.trimmed_reads
+[[ -e "../02.trimmed_reads/$b.1.fastq_trimmed.segments" ]] \
+   && rm ../02.trimmed_reads/$b.[12].fastq_trimmed.segments
+[[ -e "../02.trimmed_reads/$b.1.fastq.trimmed.paired" ]] \
+   && rm ../02.trimmed_reads/$b.[12].fastq.trimmed.paired
+[[ -e "../02.trimmed_reads/$b.1.fastq.trimmed.single" ]] \
+   && rm ../02.trimmed_reads/$b.[12].fastq.trimmed.single
+[[ -e "../02.trimmed_reads/$b.1.fastq.trimmed" ]] \
+   && rm ../02.trimmed_reads/$b.[12].fastq.trimmed
+[[ -e "../02.trimmed_reads/$b.1.fastq" ]] \
+   && rm ../02.trimmed_reads/$b.[12].fastq
+
+# Finalize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.done"
+miga add_result -P "$PROJECT" -D "$DATASET" -r read_quality
+

data/scripts/ssu.bash

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+DIR="$PROJECT/data/07.annotation/01.function/02.ssu"
+[[ -d "$DIR" ]] || mkdir -p "$DIR"
+cd "$DIR"
+
+# Initialize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.start"
+
+fa="../../../05.assembly/$DATASET.LargeContigs.fna"
+if [[ -s $fa ]] ; then
+   # Run barrnap
+   barrnap --quiet --threads $CORES $fa | grep "^##gff\\|;product=16S " \
+      > $DATASET.ssu.gff 
+
+   # Extract
+   bedtools getfasta -s "-fi" $fa -bed $DATASET.ssu.gff -fo $DATASET.ssu.all.fa 
+   FastA.length.pl $DATASET.ssu.all.fa | sort -nr -k 2 | head -n 1 \
+      | cut -f 1 > $DATASET.ssu.fa.id 
+   FastA.filter.pl $DATASET.ssu.fa.id $DATASET.ssu.all.fa > $DATASET.ssu.fa 
+   rm $DATASET.ssu.fa.id 
+   [[ -e "$fa.fai" ]] && rm "$fa.fai"
+
+   # Gzip
+   gzip -9 -f "$DATASET.ssu.gff"
+   gzip -9 -f "$DATASET.ssu.all.fa"
+fi
+
+# Finalize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.done"
+miga add_result -P "$PROJECT" -D "$DATASET" -r ssu
+

data/scripts/subclades.bash

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+cd "$PROJECT/data/10.clades/02.ani"
+
+# Initialize
+date "+%Y-%m-%d %H:%M:%S %z" > "miga-project.start"
+
+# Run R code
+echo "
+source('$MIGA/utils/subclades.R');
+subclades('../../09.distances/03.ani/miga-project.txt.gz',
+   'miga-project', $CORES);
+" | R --vanilla
+
+# Compile
+ruby "$MIGA/utils/subclades-compile.rb" . \
+   >  miga-project.class.tsv \
+   2> miga-project.class.nwk
+
+# Finalize
+date "+%Y-%m-%d %H:%M:%S %z" > "miga-project.done"
+miga add_result -P "$PROJECT" -r subclades
+

data/scripts/trimmed_fasta.bash

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+cd "$PROJECT/data/04.trimmed_fasta"
+
+b=$DATASET
+
+# Initialize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.start"
+
+# Gunzip (if necessary)
+for sis in 1 2 ; do
+   [[ -e "../02.trimmed_reads/$b.$sis.clipped.fastq.gz" \
+      && ! -e "../02.trimmed_reads/$b.$sis.clipped.fastq" ]] \
+      && gunzip "../02.trimmed_reads/$b.$sis.clipped.fastq.gz"
+done
+
+# FastQ -> FastA
+cat ../02.trimmed_reads/$b.1.clipped.fastq | FastQ.toFastA.awk > $b.1.fasta
+if [[ -e ../02.trimmed_reads/$b.2.clipped.fastq ]] ; then
+   cat ../02.trimmed_reads/$b.2.clipped.fastq | FastQ.toFastA.awk > $b.2.fasta
+   FastA.interpose.pl $b.CoupledReads.fa $b.[12].fasta
+   gzip -9 -f $b.2.fasta
+   gzip -9 -f $b.1.fasta
+   cat ../02.trimmed_reads/$b.[12].clipped.single.fastq | FastQ.toFastA.awk \
+      > $b.SingleReads.fa
+   gzip -9 -f $b.SingleReads.fa
+else
+   mv $b.1.fasta $b.SingleReads.fa
+fi
+
+# Compress input at 01.raw_reads and 02.trimmed_reads
+for sis in 1 2 ; do
+   [[ -e "../01.raw_reads/$b.$sis.fastq" ]] \
+      && gzip -9 -f "../01.raw_reads/$b.$sis.fastq"
+   [[ -e "../02.trimmed_reads/$b.$sis.clipped.fastq" ]] \
+      && gzip -9 -f "../02.trimmed_reads/$b.$sis.clipped.fastq"
+   [[ -e "../02.trimmed_reads/$b.$sis.clipped.single.fastq" ]] \
+      && gzip -9 -f "../02.trimmed_reads/$b.$sis.clipped.single.fastq"
+done
+
+# Finalize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.done"
+miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_fasta
+

data/scripts/trimmed_reads.bash

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+cd "$PROJECT/data/02.trimmed_reads"
+
+b=$DATASET
+
+# Initialize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.start"
+
+# Unzip (if necessary)
+[[ -e ../01.raw_reads/$b.1.fastq.gz && ! -e ../01.raw_reads/$b.1.fastq ]] \
+   && gunzip ../01.raw_reads/$b.1.fastq.gz
+[[ -e ../01.raw_reads/$b.2.fastq.gz && ! -e ../01.raw_reads/$b.2.fastq ]] \
+   && gunzip ../01.raw_reads/$b.2.fastq.gz
+
+# Clean existing files
+exists $b.[12].* && rm $b.[12].*
+
+# Tag
+FastQ.tag.rb -i ../01.raw_reads/$b.1.fastq -p "$b-" -s "/1" -o $b.1.fastq
+[[ -e ../01.raw_reads/$b.2.fastq ]] \
+   && FastQ.tag.rb -i ../01.raw_reads/$b.2.fastq -p "$b-" -s "/2" -o $b.2.fastq
+
+# Trim
+SolexaQA++ dynamictrim $b.[12].fastq -h 20 -d .
+SolexaQA++ lengthsort $b.[12].fastq.trimmed -l 50 -d .
+
+# Clean adapters
+if [[ -e $b.2.fastq.trimmed.paired ]] ; then
+   scythe -a $MIGA/utils/adapters.fa $b.1.fastq.trimmed.paired \
+      > $b.1.clipped.all.fastq
+   scythe -a $MIGA/utils/adapters.fa $b.2.fastq.trimmed.paired \
+      > $b.2.clipped.all.fastq
+   SolexaQA++ lengthsort $b.[12].clipped.all.fastq -l 50 -d .
+   rm $b.[12].clipped.all.fastq
+   [[ -e $b.1.clipped.all.fastq.single ]] \
+      && mv $b.1.clipped.all.fastq.single $b.1.clipped.single.fastq
+   [[ -e $b.2.clipped.all.fastq.single ]] \
+      && mv $b.2.clipped.all.fastq.single $b.2.clipped.single.fastq
+   mv $b.1.clipped.all.fastq.paired $b.1.clipped.fastq
+   mv $b.2.clipped.all.fastq.paired $b.2.clipped.fastq
+   rm $b.1.clipped.all.fastq.summary.txt &>/dev/null
+else
+   scythe -a $MIGA/utils/adapters.fa $b.1.fastq.trimmed.single \
+      > $b.1.clipped.all.fastq
+   SolexaQA++ lengthsort $b.1.clipped.all.fastq -l 50 -d .
+   mv $b.1.clipped.all.fastq.single $b.1.clipped.fastq
+fi
+rm $b.[12].*.discard &>/dev/null
+
+# Finalize
+date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.done"
+miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_reads
+