miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Manifest/categories.json

@@ -0,0 +1,132 @@
+{
+  "categories": {
+    "Sequence similarity search": {
+      "Statistics": [
+        "BlastPairwise.AAsubs.pl",
+        "BlastTab.advance.bash",
+        "BlastTab.recplot2.R",
+        "BlastTab.seqdepth.pl",
+        "BlastTab.seqdepth_nomedian.pl",
+        "BlastTab.seqdepth_ZIP.pl",
+        "BlastTab.sumPerHit.pl"
+      ],
+      "Manipulation": [
+        "BlastTab.addlen.rb",
+        "BlastTab.best_hit_sorted.pl",
+        "BlastTab.catsbj.pl",
+        "BlastTab.cogCat.rb",
+        "BlastTab.filter.pl",
+        "BlastTab.kegg_pep2path_rest.pl",
+        "BlastTab.pairedHits.rb",
+        "BlastTab.subsample.pl",
+        "BlastTab.taxid2taxrank.pl",
+        "BlastTab.topHits_sorted.rb"
+      ],
+      "Execution": [
+        "aai.rb",
+        "ani.rb",
+        "rbm.rb"
+      ]
+    },
+    "Sequence analyses": {
+      "Statistics": [
+        "FastA.gc.pl",
+        "FastA.length.pl",
+        "FastA.N50.pl",
+        "FastA.qlen.pl"
+      ],
+      "Manipulation": [
+        "FastA.filter.pl",
+        "FastA.filterLen.pl",
+        "FastA.filterN.pl",
+        "FastA.fragment.rb",
+        "FastA.interpose.pl",
+	"FastA.per_file.pl",
+        "FastA.rename.pl",
+        "FastA.revcom.pl",
+        "FastA.slider.pl",
+        "FastA.split.pl",
+        "FastA.subsample.pl",
+        "FastA.tag.rb",
+        "FastA.wrap.rb",
+        "FastQ.filter.pl",
+        "FastQ.interpose.pl",
+        "FastQ.offset.pl",
+        "FastQ.split.pl",
+        "FastQ.tag.rb",
+        "FastQ.toFastA.awk"
+      ]
+    },
+    "Diversity": {
+      "Community": [
+        "AlphaDiversity.pl",
+        "Chao1.pl",
+        "Table.barplot.R"
+      ],
+      "Population": [
+        "VCF.SNPs.rb",
+        "VCF.KaKs.rb"
+      ]
+    },
+    "Annotation": {
+      "Database mapping": [
+        "BlastTab.kegg_pep2path_rest.pl",
+        "BlastTab.taxid2taxrank.pl",
+        "EBIseq2tax.rb",
+        "NCBIacc2tax.rb",
+        "gi2tax.rb",
+        "M5nr.getSequences.rb",
+        "RefSeq.download.bash",
+        "SRA.download.bash"
+      ],
+      "Tables": [
+        "Table.barplot.R",
+        "GenBank.add_fields.rb",
+        "MyTaxa.fragsByTax.pl",
+        "Table.df2dist.R",
+        "Table.filter.pl",
+        "Table.merge.pl",
+	"Table.replace.rb",
+        "Table.round.rb",
+        "Table.split.pl"
+      ],
+      "Search": [
+        "HMM.essential.rb",
+        "HMMsearch.extractIds.rb",
+        "ogs.annotate.rb",
+        "ogs.core-pan.rb",
+        "ogs.extract.rb",
+        "ogs.mcl.rb",
+        "ogs.stats.rb",
+        "ogs.rb"
+      ]
+    },
+    "Other data": {
+      "Phylogenetic and other distances": [
+        "CharTable.classify.rb",
+        "JPlace.distances.rb",
+        "JPlace.to_iToL.rb",
+        "Newick.autoprune.R",
+        "TRIBS.test.R",
+        "TRIBS.plot-test.R",
+        "Table.df2dist.R"
+      ],
+      "Taxonomic": [
+        "CharTable.classify.rb",
+        "EBIseq2tax.rb",
+        "NCBIacc2tax.rb",
+        "Table.barplot.R",
+        "gi2tax.rb",
+        "MyTaxa.fragsByTax.pl",
+        "MyTaxa.seq-taxrank.rb",
+        "Taxonomy.silva2ncbi.rb"
+      ],
+      "Alignments": [
+        "AAsubs.log2ratio.rb",
+        "Aln.cat.rb",
+        "Aln.convert.pl",
+        "BlastPairwise.AAsubs.pl"
+      ]
+    }
+  }
+}

data/utils/enveomics/Manifest/examples.json

@@ -0,0 +1,154 @@
+{
+  "_": "Input files and directories are included in the 'Tests' folder.",
+  "examples": [
+    {
+      "_": "== Examples of genome comparisons ==",
+      "task": "ogs.stats.rb",
+      "description": ["Statistics on the groups of orthology in the Primate",
+        "Lentivirus Group, including HIV-1, HIV-2, and SIV."],
+      "values": ["primate_lentivirus.ogs",null,null,null,null,null]
+    },
+    {
+      "task": "ani.rb",
+      "description": ["Average Nucleotide Identity (ANI) between two strains",
+        "of Mycoplasma genitalium (M2288 and M2321)."],
+      "values": ["Mgen_M2288.fna","Mgen_M2321.fna",null,null,null,null,null,
+        null,null,null,null,null,null,null,null,null,null,null,null,null,null,
+        null,null,null]
+    },
+    {
+      "task": "aai.rb",
+      "description": ["Average Amino acid Identity (AAI) between Mycoplasma",
+        "genitalium (Bacteria) and Nanoarchaeum equitans (Archaea)."],
+      "values": ["Mgen_M2288.faa","Nequ_Kin4M.faa",null,null,null,null,null,
+        null,null,null,null,null,null,null,null,null,null,null,null,null,null,
+        null,null,null]
+    },
+    {
+      "task": "rbm.rb",
+      "description": ["Reciprocal Best Matches between the proteomes of the",
+        "two major HIV types (HIV-1 and HIV-2)."],
+      "values": ["hiv1.faa","hiv2.faa",null,null,null,null,null,null,null,null,
+        null,null,"hiv1-hiv2.rbm"]
+    },
+    {
+      "task": "ogs.mcl.rb",
+      "description": ["Groups of orthology in the Primate Letivirus Group,",
+         "including HIV-1, HIV-2, and SIV."],
+      "values": ["primate_lentivirus.ogs","primate_lentivirus.rbm",null,null,
+        null,null,null,null,null,null,null,null]
+    },
+    {
+      "task": "Table.df2dist.R",
+      "description": ["Transforms a list of AAI values between Xanthomonas",
+        "oryzae genomes into a distance matrix."],
+      "values": ["Xanthomonas_oryzae.aai.tsv",null,null,null,null,100.0,
+        "Xanthomonas_oryzae.aai-mat.tsv"]
+    },
+    {
+      "_": "== Recruitment plots",
+      "task": "BlastTab.catsbj.pl",
+      "description": ["Prepares recruitment plot files for a comparison",
+        "between a virome containing HIV and the HIV-1 genome."],
+      "values": [null,null,null,null,"hiv1.fna","hiv_mix-hiv1.blast.tsv"]
+    },
+    {
+      "task": "BlastTab.recplot2.R",
+      "description": ["Generates recruitment plots for a comparison",
+        "between a virome containing HIV and the HIV-1 genome."],
+      "values": ["hiv_mix-hiv1.blast.tsv",50,100,null,null,null,null,null,
+        "hiv_mix-hiv1.Rdata","hiv_mix-hiv1.pdf",null,null]
+    },
+    {
+      "_": "== Examples of functional annotations ==",
+      "task": "HMM.essential.rb",
+      "description": ["Typical single-copy bacterial genes present in",
+        "Mycoplasma genitalium."],
+      "values": ["Mgen_M2288.faa",null,null,null,true,null,null,null,null,null,
+        null,null,null,null,null,null]
+    },
+    {
+      "task": "HMM.essential.rb",
+      "description": ["Typical single-copy archaeal genes present in",
+        "Nanoarchaeum equitans."],
+      "values": ["Mgen_M2288.faa",null,null,null,null,true,null,null,null,null,
+        null,null,null,null,null,null]
+    },
+    {
+      "task": "Newick.autoprune.R",
+      "description": ["Prune an AlkB tree with 110 tips to get only distant",
+        "representatives (41)."],
+      "values": ["alkB.nwk",0.9,null,null,null,null,null,"alkB-pruned.nwk"]
+    },
+    {
+      "_": "== Examples of BLAST statistics and manipulation",
+      "task": "BlastTab.topHits_sorted.rb",
+      "description": ["Extract the best match of metagenome-derived proteins",
+        "(from the 'A metagenome') against a Gene Ontology collection."],
+      "values": ["sort","a_mg.cds-go.blast.tsv",null,null,null,null,1,null,null,
+        null,"a_mg.cds-go.blast-bm.tsv"]
+    },
+    {
+      "task": "BlastTab.sumPerHit.pl",
+      "description": ["Count the number of reads per gene in a mapping of a",
+        "metagenome to a metagenome-derived genes (from the 'A metagenome')."],
+      "values": [null,null,null,null,null,null,null,"a_mg.reads-cds.blast.tsv",
+        null,"a_mg.reads-cds.counts.tsv"]
+    },
+    {
+      "task": "BlastTab.sumPerHit.pl",
+      "description": ["Estimate the total abundance of Gene Ontology",
+        "annotations in the A metagenome, using metagenome-derived proteins,",
+        "and normalizing by the read counts of each protein."],
+      "values": ["a_mg.reads-cds.counts.tsv",null,null,null,null,true,null,
+        "a_mg.cds-go.blast.tsv",null,"a_mg.go.read-counts.tsv"]
+    },
+    {
+      "_": "== Examples of diversity ==",
+      "task": "Table.barplot.R",
+      "description": ["Barplot with the distribution of bacterial phyla in",
+        "four different sites, with taxa sorted by variance."],
+      "values": ["phyla_counts.tsv","250,100,75,200",null,null,null,null,null,
+        null,true,"var",2,null,null,"phyla_counts.pdf",10,null]
+    },
+    {
+      "task": "Chao1.pl",
+      "description": ["Phylum-richness estimated by the Chao1 index with 95%",
+        "confidence, using the distributions of bacterial phyla in four",
+        "different sites."],
+      "values": ["phyla_counts.tsv",null,1,null,null,true,null,
+         "phyla_chao1.tsv"]
+    },
+    {
+      "task": "AlphaDiversity.pl",
+      "description": ["Phylum-diversity estimated by the indices of Shannon",
+        "(H'), Inverse Simpson (1/Lambda), and true diversity of order 1 (1D),",
+        "using the distributions of bacterial phyla in four different sites."],
+      "values": ["phyla_counts.tsv",null,1,null,null,true,null,true,1,null,
+         "phyla_diversity.tsv"]
+    },
+    {
+      "_": "== Other miscelaneous examples ==",
+      "task": "CharTable.classify.rb",
+      "description": ["Classification of anthrax genomes based on can-SNPs, as",
+        "described in Van Ert 2007 (PLoS ONE 2(5):e461)."],
+      "values": ["anthrax-cansnp-data.tsv","anthrax-cansnp-key.tsv",
+        "anthrax-cansnp-classif.tsv","anthrax-cansnp-classif.nwk",null]
+    },
+    {
+      "task": "TRIBS.test.R",
+      "description": ["Test overclustering of Xanthomonas oryzae genomes",
+        "encoding for PilA using Transformed-space Resampling In Biased Sets",
+        "(TRIBS)."],
+      "values": ["Xanthomonas_oryzae.aai-mat.tsv","Xanthomonas_oryzae-PilA.txt",
+        5000,null,null,null,null,0,"Xanthomonas_oryzae-PilA.tribs.Rdata",100]
+    },
+    {
+      "task": "TRIBS.plot-test.R",
+      "description": ["Show the TRIBS-normalized distances between Xanthomonas",
+        "oryzae genomes (grey) and X. oryzae encoding for PilA (red)."],
+      "values": ["Xanthomonas_oryzae-PilA.tribs.Rdata",null,null,null,null,null,
+        null,null,"Xanthomonas_oryzae-PilA.tribs.pdf",null,null]
+    }
+  ]
+}

data/utils/enveomics/Manifest/tasks.json

@@ -0,0 +1,4 @@
+{
+  "_": "This file loads all the .json files inside 'Manifest/Tasks'.",
+  "_include": "Tasks/*.json"
+}

data/utils/enveomics/Pipelines/assembly.pbs/CONFIG.mock.bash

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+##################### VARIABLES
+# Queue: Preferred queue.  Delete (or comment) this line to allow
+# automatic detection:
+#QUEUE="biocluster-6"
+# If you set the QUEUE variable, you MUST set the WTIME variable
+# as well, containing the walltime to be asked for.  The WTIME
+# variable is ignored otherwise.
+WTIME="120:00:00"
+
+# Scratch:  This is where the output will be created.
+SCRATCH="$HOME/scratch/pipelines/assembly"
+
+# Data folder:  This is the folder that cointains the input files.
+DATA="$HOME/data/trim"
+
+# Location of Newbler's binaries
+BIN454="$HOME/454/bin"
+
+# Name(s) of the library(ies) to use, separated by spaces:
+# This is determined by the name of your input files.  For example,
+# if your input files are: LLSEP.CoupledReads.fa and LWP.CoupledReads.fa,
+# use:
+# LIBRARIES="LLSEP LWP"
+# It's strongly encouraged to use only one per CONFIG file.
+LIBRARIES="A";
+
+# Use .CoupledReads.fa and/or .SingleReads.fa (yes or no):
+USECOUPLED=yes
+USESINGLE=no
+
+# Insert length (in bp):  This is the average length of the entire insert,
+# not just the gap length.
+INSLEN=300
+
+# Number of CPUs to use (for SOAP and Newbler):
+PPN=16
+
+# RAM multiplier: Multiply the estimated required RAM by this number:
+RAMMULT=1
+
+# Maximum number of simultaneous jobs: Uncomment and increase these values if
+# you have increased resources (e.g., a dedicated queue); uncomment and decrease
+# if the resources are scarce (e.g., a very busy queue or other simultaneous jobs).
+#VELVETSIM=22
+#SOAPSIM=8
+
+# Extra parameters for Velvet: Any additional parameters to be passed to
+# velvetg or velveth.  If you have MP data, consider adding the option
+# -shortMatePaired yes to VELVETG_EXTRA.  If you have Nextera, consider
+# adding the option above, plus the option -ins_length_sd <integer>, to
+# indicate the standard deviation of the insert size.  By default, the
+# SD is assumed to be 10% of the average, but Nextera produces much
+# wider distribution of sizes (i.e., larger SD).  Typically you shouldn't
+# need to add anything in VELVETH_EXTRA.
+VELVETH_EXTRA=""
+VELVETG_EXTRA=""
+
+# Clean non-essential files (yes or no):
+CLEANUP=yes
+
+# Best k-mers:  Space-delimited list of kmers selected from Velvet and SOAP.
+# This is to be modified at the begining of step 4, and it's ignored in all
+# the other steps.
+K_VELVET="21 23 35"
+K_SOAP="21 23 35"
+
+

data/utils/enveomics/Pipelines/assembly.pbs/FastA.N50.pl

@@ -0,0 +1,56 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Oct 07 2015
+# @license: artistic license 2.0
+#
+use strict;
+use warnings;
+use List::Util qw/sum min max/;
+
+my ($seqs, $minlen, $n__) = @ARGV;
+$seqs or die "
+Description:
+   Calculates the N50 value of a set of sequences.  Alternatively, it
+   can calculate other N** values.  It also calculates the total number
+   of sequences and the total added length.
+   
+Usage:
+   $0 seqs.fa[ minlen[ **]]
+
+   seqs.fa	A FastA file containing the sequences.
+   minlen	(optional) The minimum length to take into consideration.
+   		By default: 0.
+   **		Value N** to calculate.  By default: 50 (N50).
+";
+$minlen ||= 0;
+$n__    ||= 50;
+
+my @len = ();
+open SEQ, "<", $seqs or die "Cannot open file: $seqs: $!\n";
+while(<SEQ>){
+   if(/^>/){
+      push @len, 0;
+   }else{
+      next if /^;/;
+      chomp;
+      s/\W//g;
+      $len[-1]+=length $_;
+   }
+}
+close SEQ;
+@len = sort { $a <=> $b } map { $_>=$minlen?$_:() } @len;
+my $tot = (sum(@len) || 0);
+
+my $thr = $n__*$tot/100;
+my $pos = 0;
+for(@len){
+   $pos+= $_;
+   if($pos>=$thr){
+      print "N$n__: $_\n";
+      last;
+   }
+}
+print "Sequences: ".scalar(@len)."\n";
+print "Total length: $tot\n";
+

data/utils/enveomics/Pipelines/assembly.pbs/FastA.filterN.pl

@@ -0,0 +1,60 @@
+#!/usr/bin/env perl
+# 
+# @author  Luis M. Rodriguez-R
+# @update  Oct-07-2015
+# @license artistic license 2.0
+# 
+
+use warnings;
+use strict;
+
+my($file, $content, $stretch) = @ARGV;
+$file or die <<HELP
+
+Description:
+   Filter sequences by N-content and presence of long homopolymers.
+Usage:
+   $0 sequences.fa [content [stretch]] > filtered.fa
+Where:
+   sequences.fa	Input file in FastA format
+   content	A number between 0 and 1 indicating the maximum proportion of Ns
+   		(1 to turn off, 0.5 by default)
+   stretch	A number indicating the maximum number of consecutive identical
+   		nucleotides allowed (0 to turn off, 100 by default)
+   filtered.fa	Filtered set of sequences.
+
+HELP
+;
+($content ||= 0.5)+=0;
+($stretch ||= 100)+=0;
+
+my $good = 0;
+my $N = 0;
+
+FASTA: {
+   local $/ = "\n>";
+   open FILE, "<", $file or die "I can not open the file: $file: $!\n";
+   SEQ: while(<FILE>){
+      $N++;
+      s/^;.*//gm;
+      s/>//g;
+      my($n,$s) = split /\n/, $_, 2;
+      (my $clean = $s) =~ s/[^ACTGN]//g;
+      if($content < 1){
+         (my $Ns = $clean) =~ s/[^N]//g;
+	 next SEQ if length($Ns)>length($clean)*$content;
+      }
+      if($stretch > 0){
+         for my $nuc (qw(A C T G N)){
+	    next SEQ if $clean =~ m/[$nuc]{$stretch}/;
+	 }
+      }
+      print ">$n\n$s\n";
+      $good++;
+   }
+   close FILE;
+   print STDERR "Total sequences: $N\nAfter filtering: $good\n";
+}
+
+
+