miga-base 0.7.26.0 → 1.0.0.1

data/utils/distance/commands.rb

@@ -169,6 +169,7 @@ module MiGA::DistanceRunner::Commands
         aai_data[out[1]] = [out[6].to_f, 0, 0, 0] if out[6] !~ /^>/
       end
     end
+    puts "Results: #{haai_data.size} | Inferences: #{aai_data.size}"
     batch_data_to_db(:haai, haai_data)
     batch_data_to_db(:aai, aai_data)
 

data/utils/distance/database.rb

@@ -126,7 +126,6 @@ module MiGA::DistanceRunner::Database
   def batch_data_to_db(metric, data)
     db = tmp_dbs[metric]
     table = metric == :haai ? :aai : metric
-    `cp #{db} ~/here.db`
     SQLite3::Database.new(db) do |conn|
       data.each do |k, v|
         sql = <<~SQL

data/utils/distance/runner.rb

@@ -18,10 +18,8 @@ class MiGA::DistanceRunner
       @ref_project = MiGA::Project.load(ref_path)
       raise "Cannot load reference project: #{ref_path}" if @ref_project.nil?
     elsif !opts[:run_taxonomy] && dataset.option(:db_project)
-      ref_path = dataset.option(:db_project)
-      if project.option(:db_proj_dir)
-        ref_path = File.expand_path(ref_path, project.option(:db_proj_dir))
-      end
+      ref_location = project.option(:db_proj_dir) || File.dirname(project.path)
+      ref_path = File.expand_path(dataset.option(:db_project), ref_location)
       @ref_project = MiGA::Project.load(ref_path)
       raise "Cannot load reference project: #{ref_path}" if @ref_project.nil?
     else

data/utils/enveomics/Docs/recplot2.md

@@ -0,0 +1,244 @@
+# Recruitment plots
+
+## Aims
+
+This document aims to cover the technical aspects of the recruitment plot functions in the
+`enveomics.R` package, focusing on the peak finder and gene-content diversity analyses.
+
+## Caveats
+
+This is a __*working document*__, describing  unstable and/or experimental code. The material
+here is susceptible of changes without warning, pay attention to the modification date and (if
+in doubt) the commit history. The definitions and default parameters of the functions described
+here may change in the near future as result of further experimentation or more stable
+implementations.
+
+The current document was generated and tested with the `enveomics.R` package version 1.3. To
+check your current version in R, use `packageVersion('enveomics.R')`.
+
+> **IMPORTANT**: Some of the functions described here may return unexpected results with your data.
+> Carefully evaluate all your results.
+
+---
+
+## Package: `enveomics.R`
+
+The functionalities described here are provided by the `enveomics.R` package. Some features
+described here are updated more frequently than the official
+[CRAN releases](https://CRAN.R-project.org/package=enveomics.R). In order to have the latest
+updates (package HEAD), download (or update), and install this git repository.
+
+### Quick installation guide
+
+:globe_with_meridians: To install the latest stable version available in CRAN, use in R:
+
+```R
+install.packages(c('enveomics.R','optparse'))
+```
+
+:octocat: To install the latest HEAD version (potentially unstable) available in GitHub, use in R:
+
+```R
+install.packages('devtools')
+library('devtools')
+install_github('lmrodriguezr/enveomics', subdir='enveomics.R')
+```
+
+---
+
+## Recruitment plots: `enve.recplot2`
+
+The first step in this analysis is the mapping of reads to the genome, processed with
+[BlastTab.catsbj.pl](http://enve-omics.ce.gatech.edu/enveomics/docs?t=BlastTab.catsbj.pl).
+We'll assume the mapping is saved in the file `my-mapping.tab` and this is also the
+prefix of the processed files.
+
+Once you have these input files (`.rec` and `.lim`), you can build the recruitment plot.
+For this, you'll have two options.
+
+### Option 1: Using the `BlastTab.recplot2.R` stand-alone script
+
+The stand-alone script
+[BlastTab.recplot2.R](http://enve-omics.ce.gatech.edu/enveomics/docs?t=BlastTab.recplot2.R)
+is the easiest option to run, and should be the preferred method if you're automating
+this analysis to process several mappings, but it doesn't offer access to advanced options.
+
+You can run it like this using two CPUs:
+
+```bash
+BlastTab.recplot2.R --prefix my-mapping.tab --threads 2 my-recplot.rdata my-recplot.pdf
+```
+
+> **NOTE 1**: It's NOT recommended to map reads against genes, the recommended strategy is to
+> map against contigs. However, if you did map reads against genes, you may want to use the
+> `--pos-breaks 0` option to use each gene as a recruitment window.
+> 
+> **NOTE 2**: If you want to plot the population peaks at this step, simply pass the
+> `--peaks-col darkred` option.
+
+Now you should have two output files: `my-recplot.rdata`, containing your `enve.RecPlot2` R
+object, and `my-recplot.pdf` with the graphical output of the recruitment plot.
+
+### Option 2: Using the `enve.recplot2` R function
+
+If you require access to advanced options, or for some other reason prefer to calculate the
+recruitment plot interactively, you can directly use the `enve.recplot2` R function. This is
+and example session in R:
+
+```R
+# Load the package
+library(enveomics.R)
+# Open the PDF
+pdf('my-recplot.pdf')
+# Build and plot the object using two threads and no peak detection
+# (to turn on peak detection, simply remove `peaks.col=NA`)
+rp <- enve.recplot2('my-mapping.tab', threads=2, peaks.col=NA)
+# Close the PDF
+dev.off()
+# Save the object
+save(rp, file='my-recplot.rdata')
+```
+
+> **IMPORTANT**: Remember to save the `enve.RecPlot2` R object (that's the last line above)
+> before closing the R session.
+
+Naturally, you may want to see what other (advanced) options you have. You can access the
+documentation of the function in R using `?enve.recplot2`.
+
+---
+
+## Summary statistics
+
+Here we explore some frequently used summary statistics from recruitment plots. First, load the
+package and the `enve.RecPlot2` object you saved previously, in R:
+
+```R
+library(enveomics.R)
+load('my-recplot.rdata')
+```
+
+### Centrality measures of sequencing depth
+
+```R
+mean(enve.recplot2.seqdepth(rp)) # <- Average
+median(enve.recplot2.seqdepth(rp)) # <- Median
+enve.truncate(enve.recplot2.seqdepth(rp)) # <- 95% Central Truncated Mean
+enve.truncate(enve.recplot2.seqdepth(rp), 0.9) # <- 90% Central Truncated Mean
+```
+
+The functions above only use hits with identity above the cutoff for "in-group" (by default: 95%).
+In order to estimate the sequencing depth with a different identity cutoff, modify the cutoff first:
+
+```R
+rp98 <- enve.recplot2.changeCutoff(rp, 98) # <- Change to ≥98%
+mean(enve.recplot2.seqdepth(rp98)) # <- Average (for the new object)
+median(enve.recplot2.seqdepth(rp98)) # <- Median (for the new object)
+```
+
+### Average and median sequencing depth excluding zero-coverage windows
+
+```R
+seqdepth <- enve.recplot2.seqdepth(rp)
+mean(seqdepth[seqdepth>0]) # <- Average
+median(seqdepth[seqdepth>0]) # <- Median
+```
+
+### Average Nucleotide Identity from reads (ANIr)
+
+```R
+enve.recplot2.ANIr(rp) # <- Complete recruitment plot
+enve.recplot2.ANIr(rp, c(90,100)) # <- All reads above 90% (recommended for intra-population)
+enve.recplot2.ANIr(rp, c(95,100)) # <- Reads above 95%
+enve.recplot2.ANIr(rp, c( 0, 90)) # <- Between populations (other species)
+```
+
+### Coordinates of each sequence window with their respective sequencing depth
+
+```R
+d <- enve.recplot2.coordinates(rp)
+d$seqdepth <- enve.recplot2.seqdepth(rp)
+d
+```
+
+### Sequencing breadth (upper boundary)
+
+This estimate depends on the window size. The smaller the window size, the better the
+estimate. When the window size is 1bp, the estimate is exact, otherwise it's consistently
+biased (overestimate).
+
+```R
+mean(enve.recplot2.seqdepth(rp) > 0)
+```
+
+---
+
+## Peak-finder: `enve.recplot2.findPeaks`
+
+In this step we will try to identify one or multiple population peaks corresponding to different
+sub-populations and/or composites of sub-populations.
+
+> **NOTE** This step can be performed together with the step above, but we separate it here for
+> two reasons: **(1)** This step is much more unstable but less computationally demanding than the
+> step before, so it makes sense to re-run only this part with different parameters and/or
+> package updates; and **(2)** We want to save the R objects independently, so the following steps
+> are more clear.
+
+In R:
+
+```R
+# Load the package
+library(enveomics.R)
+# Load the `enve.RecPlot2` object you saved previously
+load('my-recplot.rdata')
+# Find the peaks
+peaks <- enve.recplot2.findPeaks(rp)
+# Save the peaks R object (optional)
+save(peaks, file='my-recplot-peaks.rdata')
+# Plot the peaks in a PDF (optional)
+pdf('my-recplot-peaks.pdf')
+p <- plot(rp, use.peaks=peaks, layout=4) # <- Remove `layout=4` for the full plot
+dev.off()
+```
+
+The key function here is `enve.recplot2.findPeaks`. This function has several parameters, depending on
+the method used. To see all supported methods, use `?enve.recplot2.findPeaks`. To see all the options
+of the default method (`'emauto'`) use `?enve.recplot2.findPeaks.emauto`.
+
+---
+
+## Gene-content diversity: `enve.recplot2.extractWindows`
+
+In R:
+
+```R
+# Load the package and the objects (unless you're still in the same session from the last step)
+library(enveomics.R)
+load('my-recplot.rdata')
+load('my-recplot-peaks.rdata')
+# Find the peak representing the core genome
+cp <- enve.recplot2.corePeak(peaks)
+#-----
+# The following functions illustrate how to obtain different results. Please explore the resulting
+# objects and the associated documentation
+#-----
+# Find the coordinates of windows significantly below the average sequencing depth
+div <- enve.recplot2.extractWindows(rp, cp, seq.names=TRUE)
+# Add sequencing depth
+div$seqdepth <- enve.recplot2.seqdepth(rp, as.numeric(rownames(div)))
+# Save the coordinates as a tab-delimited table
+write.table(div, 'my-low-seqdepth.tsv', quote=FALSE, sep='\t', row.names=FALSE)
+# Find all the windows with sequencing depth zero
+zero <- enve.recplot2.coordinates(rp, enve.recplot2.seqdepth(rp)==0)
+```
+
+---
+
+## To do
+
+- [x] Document structure
+- [x] Package: `enveomics.R`
+- [x] Recruitment plots: `enve.recplot2`
+- [x] Summary statistics
+- [x] Peak-finder: `enve.recplot2.findPeaks`
+- [x] Gene-content diversity: `enve.recplot2.extractWindows`
+- [ ] Compare identity profiles: `enve.recplot2.compareIdentities`

data/utils/enveomics/Examples/aai-matrix.bash

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# @author  Luis M. Rodriguez-R
+# @license Artistic-2.0
+
+set -e # <- So it stops if there is an error
+function exists { [[ -e "$1" ]] ; } # <- To test *any* of many files
+
+OUT=$1		# <- Output file
+[[ -n "$1" ]] && shift
+SEQS=("$@")	# <- list of all genomes
+THR=2		# <- Number or threads
+DEF_DIST=0.9	# <- Default distance when AAI cannot be reliably estimated
+
+# This is just the help message
+if [[ $# -lt 2 ]] ; then
+echo "
+Use case: Building AAI matrices from a collection of genomes.
+
+IMPORTANT
+This script is functional, but it's mainly intended for illustrative purposes.
+Please take a look at the code first.
+
+Usage:
+$0 <output.txt> <genomes...>
+
+<output.txt>	The output AAI list, in tab-delimited form containing the
+		following columns: (1) Sequence A, (2) Sequence B, (3)
+		AAI, (4) AAI-SD, (5) Proteins used, (6) Number of proteins in
+		the smallest genome, (7) Percentage of the genome shared.
+<genomes...>	The list of files containing the genomes (at least 2).
+
+" >&2
+exit
+fi
+
+# 00. Create environment
+export PATH=$(dirname "$0")/../Scripts:$PATH
+
+# 01. Calculate AAI
+echo "[01/03] Calculating AAI"
+for i in "${SEQS[@]}" ; do
+  for j in "${SEQS[@]}" ; do
+    echo -n " o $i vs $j: "
+    AAI=$(aai.rb -1 "$i" -2 "$j" -S "$OUT.db" -t "$THR" \
+      --no-save-rbm --auto --quiet)
+    echo ${AAI:-Below detection}
+    [[ "$i" == "$j" ]] && break
+  done
+done
+
+# 02. Extract matrix
+echo "[02/03] Extracting list"
+echo -e "SeqA\tSeqB\tAAI\tSD\tN\tOmega\tFrx" > "$OUT"
+echo "select seq1, seq2, aai, sd, n, omega, (100.0*n/omega) from aai;" \
+  | sqlite3 "$OUT.db" | tr '|' '\t' >> "$OUT"
+
+# 03. Make it a distance matrix.
+echo "[03/03] Generating distance matrix"
+echo "
+source('$(dirname $0)/../enveomics.R/R/df2dist.R');
+a <- read.table('$OUT', sep = '\\t', header = TRUE, as.is = TRUE, quote = '');
+aai.d <- enve.df2dist(a, default.d = $DEF_DIST, max.sim = 100);
+write.table(as.matrix(aai.d), '$OUT.dist',
+  quote = FALSE, col.names = NA, row.names = TRUE, sep = '\\t')
+" | R --vanilla >/dev/null

data/utils/enveomics/Examples/ani-matrix.bash

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# @author  Luis M. Rodriguez-R
+# @license Artistic-2.0
+
+set -e # <- So it stops if there is an error
+function exists { [[ -e "$1" ]] ; } # <- To test *any* of many files
+
+OUT=$1		# <- Output file
+[[ -n "$1" ]] && shift
+SEQS=("$@")	# <- list of all genomes
+THR=2		# <- Number or threads
+DEF_DIST=0.9	# <- Default distance when ANI cannot be reliably estimated
+
+# This is just the help message
+if [[ $# -lt 2 ]] ; then
+echo "
+Use case: Building ANI matrices from a collection of genomes.
+
+IMPORTANT
+This script is functional, but it's mainly intended for illustrative purposes.
+Please take a look at the code first.
+
+Usage:
+$0 <output.txt> <genomes...>
+
+<output.txt>	The output ANI list, in tab-delimited form containing the
+		following columns: (1) Sequence A, (2) Sequence B, (3)
+		ANI, (4) ANI-SD, (5) Fragments used, (6) Maximum number
+		of fragments, (7) Percentage of the genome shared.
+<genomes...>	The list of files containing the genomes (at least 2).
+
+" >&2
+exit
+fi
+
+# 00. Create environment
+export PATH=$(dirname "$0")/../Scripts:$PATH
+
+# 01. Calculate ANI
+echo "[01/03] Calculating ANI"
+for i in "${SEQS[@]}" ; do
+  for j in "${SEQS[@]}" ; do
+    echo -n " o $i vs $j: "
+    ANI=$(ani.rb -1 "$i" -2 "$j" -S "$OUT.db" -t "$THR" \
+      --no-save-rbm --no-save-regions --auto --quiet)
+    echo ${ANI:-Below detection}
+    [[ "$i" == "$j" ]] && break
+  done
+done
+
+# 02. Extract matrix
+echo "[02/03] Extracting list"
+echo -e "SeqA\tSeqB\tANI\tSD\tN\tOmega\tFrx" > "$OUT"
+echo "select seq1, seq2, ani, sd, n, omega, (100.0*n/omega) from ani;" \
+  | sqlite3 "$OUT.db" | tr '|' '\t' >> "$OUT"
+
+# 03. Make it a distance matrix.
+echo "[03/03] Generating distance matrix"
+echo "
+source('$(dirname $0)/../enveomics.R/R/df2dist.R');
+a <- read.table('$OUT', sep = '\\t', header = TRUE, as.is = TRUE, quote = '');
+ani.d <- enve.df2dist(a, default.d = $DEF_DIST, max.sim = 100);
+write.table(as.matrix(ani.d), '$OUT.dist',
+  quote = FALSE, col.names = NA, row.names = TRUE, sep = '\\t')
+" | R --vanilla >/dev/null

data/utils/enveomics/Examples/essential-phylogeny.bash

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+#
+# @author  Luis M. Rodriguez-R
+# @update  Mar-23-2016
+# @license artistic license 2.0
+#
+
+set -e # <- So it stops if there is an error
+function exists { [[ -e "$1" ]] ; } # <- To test *any* of many files
+
+ORG=$1 # <- Organism (see help)
+THR=2 # <- Number or threads
+
+# This is just the help message
+if [[ "$ORG" == "" ]] ; then
+echo "
+Use case: Essential genes phylogeny of a species. The essential genes are a
+collection of genes typically found in single copy in archaeal and bacterial
+genomes
+
+IMPORTANT
+This script is functional, but it's mainly intended for illustrative purposes.
+Please take a look at the code first.
+
+Usage:
+$0 <organism>
+
+<organism>	The organism to use (e.g., Streptococcus_pneumoniae).
+
+" >&2
+exit
+fi
+
+# 00. Create environment
+export PATH=$(dirname $0)/../Scripts:$PATH
+if [[ -e $ORG ]] ; then
+   echo "Cowardly refusing to overwrite $ORG, please remove archive first." >&2
+   exit 1
+fi
+mkdir $ORG
+for i in 01.proteome 02.essential 03.aln 04.cat 05.raxml 06.autoprune ; do
+   mkdir $ORG/$i
+done
+
+# 01. Download proteomes
+echo "[01/06] Downloading and guzipping data"
+RefSeq.download.bash $ORG .faa.gz "Complete Genome" $ORG/01.proteome
+rm $ORG/01.proteome/assembly_summary.txt
+for i in $ORG/01.proteome/* ; do
+   b=$(basename $i | perl -pe 's/[^A-Za-z0-9]/_/g' | perl -pe 's/_+$//')
+   if exists $i/*.faa.gz ; then
+      for j in $i/*.faa.gz ; do gunzip $j ; done
+      cat $i/*.faa > $ORG/01.proteome/$b.faa
+   fi
+   rm -R $i
+done
+
+# 02. Essential genes
+echo "[02/06] Idenfifying essential genes"
+N=0
+for i in $ORG/01.proteome/*.faa ; do # <- This loop could be parallelized
+   genomeA=$(basename $i .faa)
+   dir=$ORG/02.essential/$genomeA
+   mkdir $dir
+   HMM.essential.rb -i $i -m $dir/ -R $dir/log.txt -r $genomeA -t $THR
+   let N=$N+1
+done
+
+# 03. Find core and align groups
+echo "[03/06] Identifying core essentials and aligning groups"
+CORE_ESS=$(basename -s .faa $ORG/02.essential/*/*.faa | sort | uniq -c \
+   | awk '$1=='$N'{print $2}') 
+for b in $CORE_ESS ; do # <- This loop could be parallelized
+   cat $ORG/02.essential/*/$b.faa > $ORG/03.aln/$b.faa
+   clustalo -i $ORG/03.aln/$b.faa -o $ORG/03.aln/$b.aln #--threads=$THR
+done
+
+# 04. Concatenate alignment
+echo "[04/06] Concatenating alignments and removing invariable sites"
+Aln.cat.rb -I -c $ORG/04.cat/essential.raxcoords -i '|' $ORG/03.aln/*.aln \
+   > $ORG/04.cat/essential.aln 2> $ORG/04.cat/essential.log
+
+# 05. Run RAxML
+echo "[05/06] Inferring phylogeny"
+# You REALLY should consider running the following with more threads (-T) and,
+# if possible, multi-nodes using MPI
+cd $ORG/05.raxml
+raxmlHPC-PTHREADS -T $THR -p 1234 \
+   -s ../04.cat/essential.aln -q ../04.cat/essential.raxcoords \
+   -m PROTCATGTR -n UNUS #  IMPORTANT:	Please read the documentation of RAxML
+   			 # 		before running this line, so you know
+			 #  that you're running what you really want. Check
+			 #  options for bootstrapping and the different
+			 #  algorithms (-f). Note that -m is required, but the
+			 #  file unus.raxcoords specifies "AUTO", so RAxML will
+			 #  attempt to find the model resulting in the highest
+			 #  likelihood.
+cd ../..
+
+# 06. Autoprune
+echo "[06/06] Auto-pruning the tree"
+Newick.autoprune.R --t $ORG/05.raxml/RAxML_bestTree.UNUS --min_dist 0.001 \
+   $ORG/06.autoprune/essential-pruned.nwk
+