miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Manifest/categories.json

@@ -0,0 +1,165 @@
+{
+  "categories": {
+    "Sequence similarity search": {
+      "Statistics": [
+        "BedGraph.tad.rb",
+        "BedGraph.window.rb",
+        "BlastPairwise.AAsubs.pl",
+        "BlastTab.advance.bash",
+        "BlastTab.recplot2.R",
+        "BlastTab.seqdepth.pl",
+        "BlastTab.seqdepth_nomedian.pl",
+        "BlastTab.seqdepth_ZIP.pl",
+        "BlastTab.sumPerHit.pl",
+        "FastQ.test-error.rb",
+        "RecPlot2.compareIdentities.R"
+      ],
+      "Manipulation": [
+        "BlastTab.addlen.rb",
+        "BlastTab.best_hit_sorted.pl",
+        "BlastTab.catsbj.pl",
+        "BlastTab.cogCat.rb",
+        "BlastTab.filter.pl",
+        "BlastTab.kegg_pep2path_rest.pl",
+        "BlastTab.pairedHits.rb",
+        "BlastTab.subsample.pl",
+        "BlastTab.taxid2taxrank.pl",
+        "BlastTab.topHits_sorted.rb",
+        "sam.filter.rb"
+      ],
+      "Execution": [
+        "aai.rb",
+        "ani.rb",
+        "anir.rb",
+        "HMM.haai.rb",
+        "rbm.rb"
+      ]
+    },
+    "Sequence analyses": {
+      "Statistics": [
+        "FastA.gc.pl",
+        "FastA.length.pl",
+        "FastA.N50.pl",
+        "FastA.qlen.pl",
+        "FastQ.test-error.rb"
+      ],
+      "Manipulation": [
+        "FastA.extract.rb",
+        "FastA.filter.pl",
+        "FastA.filterLen.pl",
+        "FastA.filterN.pl",
+        "FastA.fragment.rb",
+        "FastA.interpose.pl",
+        "FastA.mask.rb",
+        "FastA.per_file.pl",
+        "FastA.rename.pl",
+        "FastA.revcom.pl",
+        "FastA.sample.rb",
+        "FastA.slider.pl",
+        "FastA.split.pl",
+        "FastA.split.rb",
+        "FastA.subsample.pl",
+        "FastA.tag.rb",
+        "FastA.toFastQ.rb",
+        "FastA.wrap.rb",
+        "FastQ.filter.pl",
+        "FastQ.interpose.pl",
+        "FastQ.maskQual.rb",
+        "FastQ.offset.pl",
+        "FastQ.split.pl",
+        "FastQ.tag.rb",
+        "FastQ.toFastA.awk"
+      ]
+    },
+    "Diversity": {
+      "Community": [
+        "AlphaDiversity.pl",
+        "Chao1.pl",
+        "Table.barplot.R",
+        "Table.prefScore.R"
+      ],
+      "Population": [
+        "VCF.SNPs.rb",
+        "VCF.KaKs.rb",
+        "Table.prefScore.R"
+      ]
+    },
+    "Annotation": {
+      "Database mapping": [
+        "BlastTab.kegg_pep2path_rest.pl",
+        "BlastTab.taxid2taxrank.pl",
+        "EBIseq2tax.rb",
+        "NCBIacc2tax.rb",
+        "gi2tax.rb",
+        "M5nr.getSequences.rb",
+        "RefSeq.download.bash",
+        "SRA.download.bash"
+      ],
+      "Tables": [
+        "Table.barplot.R",
+        "GenBank.add_fields.rb",
+        "MyTaxa.fragsByTax.pl",
+        "Table.df2dist.R",
+        "Table.filter.pl",
+        "Table.merge.pl",
+        "Table.replace.rb",
+        "Table.round.rb",
+        "Table.split.pl"
+      ],
+      "Search": [
+        "HMM.essential.rb",
+        "HMM.haai.rb",
+        "HMMsearch.extractIds.rb",
+        "ogs.annotate.rb",
+        "ogs.core-pan.rb",
+        "ogs.extract.rb",
+        "ogs.mcl.rb",
+        "ogs.stats.rb",
+        "ogs.rb"
+      ]
+    },
+    "Other data": {
+      "Phylogenetic and other distances": [
+        "CharTable.classify.rb",
+        "JPlace.distances.rb",
+        "JPlace.to_iToL.rb",
+        "Newick.autoprune.R",
+        "TRIBS.test.R",
+        "TRIBS.plot-test.R",
+        "Table.df2dist.R"
+      ],
+      "Taxonomic": [
+        "CharTable.classify.rb",
+        "EBIseq2tax.rb",
+        "NCBIacc2tax.rb",
+        "Table.barplot.R",
+        "gi2tax.rb",
+        "MyTaxa.fragsByTax.pl",
+        "MyTaxa.seq-taxrank.rb",
+        "Taxonomy.silva2ncbi.rb"
+      ],
+      "Alignments": [
+        "AAsubs.log2ratio.rb",
+        "Aln.cat.rb",
+        "Aln.convert.pl",
+        "BlastPairwise.AAsubs.pl"
+      ],
+      "Clustering": [
+        "ogs.mcl.rb",
+        "clust.rand.rb"
+      ],
+      "Read recruitments": [
+        "anir.rb",
+        "BedGraph.tad.rb",
+        "BedGraph.window.rb",
+        "BlastTab.catsbj.pl",
+        "BlastTab.pairedHits.rb",
+        "BlastTab.recplot2.R",
+        "FastQ.test-error.rb",
+        "GFF.catsbj.pl",
+        "RecPlot2.compareIdentities.R",
+        "sam.filter.rb"
+      ]
+    }
+  }
+}

data/utils/enveomics/Manifest/examples.json

@@ -0,0 +1,154 @@
+{
+  "_": "Input files and directories are included in the 'Tests' folder.",
+  "examples": [
+    {
+      "_": "== Examples of genome comparisons ==",
+      "task": "ogs.stats.rb",
+      "description": ["Statistics on the groups of orthology in the Primate",
+        "Lentivirus Group, including HIV-1, HIV-2, and SIV."],
+      "values": ["primate_lentivirus.ogs",null,null,null,null,null]
+    },
+    {
+      "task": "ani.rb",
+      "description": ["Average Nucleotide Identity (ANI) between two strains",
+        "of Mycoplasma genitalium (M2288 and M2321)."],
+      "values": ["Mgen_M2288.fna","Mgen_M2321.fna",null,null,null,null,null,
+        null,null,null,null,null,null,null,null,null,null,null,null,null,null,
+        null,null,null]
+    },
+    {
+      "task": "aai.rb",
+      "description": ["Average Amino acid Identity (AAI) between Mycoplasma",
+        "genitalium (Bacteria) and Nanoarchaeum equitans (Archaea)."],
+      "values": ["Mgen_M2288.faa","Nequ_Kin4M.faa",null,null,null,null,null,
+        null,null,null,null,null,null,null,null,null,null,null,null,null,null,
+        null,null,null]
+    },
+    {
+      "task": "rbm.rb",
+      "description": ["Reciprocal Best Matches between the proteomes of the",
+        "two major HIV types (HIV-1 and HIV-2)."],
+      "values": ["hiv1.faa","hiv2.faa",null,null,null,null,null,null,null,null,
+        null,null,"hiv1-hiv2.rbm"]
+    },
+    {
+      "task": "ogs.mcl.rb",
+      "description": ["Groups of orthology in the Primate Letivirus Group,",
+         "including HIV-1, HIV-2, and SIV."],
+      "values": ["primate_lentivirus.ogs","primate_lentivirus.rbm",null,null,
+        null,null,null,null,null,null,null,null]
+    },
+    {
+      "task": "Table.df2dist.R",
+      "description": ["Transforms a list of AAI values between Xanthomonas",
+        "oryzae genomes into a distance matrix."],
+      "values": ["Xanthomonas_oryzae.aai.tsv",null,null,null,null,100.0,
+        "Xanthomonas_oryzae.aai-mat.tsv"]
+    },
+    {
+      "_": "== Recruitment plots",
+      "task": "BlastTab.catsbj.pl",
+      "description": ["Prepares recruitment plot files for a comparison",
+        "between a virome containing HIV and the HIV-1 genome."],
+      "values": [null,null,null,null,"hiv1.fna","hiv_mix-hiv1.blast.tsv"]
+    },
+    {
+      "task": "BlastTab.recplot2.R",
+      "description": ["Generates recruitment plots for a comparison",
+        "between a virome containing HIV and the HIV-1 genome."],
+      "values": ["hiv_mix-hiv1.blast.tsv",50,100,null,null,null,null,null,null,
+        null,null,null,"hiv_mix-hiv1.Rdata","hiv_mix-hiv1.pdf",null,null]
+    },
+    {
+      "_": "== Examples of functional annotations ==",
+      "task": "HMM.essential.rb",
+      "description": ["Typical single-copy bacterial genes present in",
+        "Mycoplasma genitalium."],
+      "values": ["Mgen_M2288.faa",null,null,null,null,null,null,true,null,null,
+        null,null,null,null,null,null,null,null,null]
+    },
+    {
+      "task": "HMM.essential.rb",
+      "description": ["Typical single-copy archaeal genes present in",
+        "Nanoarchaeum equitans."],
+      "values": ["Mgen_M2288.faa",null,null,null,null,null,null,null,true,null,
+        null,null,null,null,null,null,null,null,null]
+    },
+    {
+      "task": "Newick.autoprune.R",
+      "description": ["Prune an AlkB tree with 110 tips to get only distant",
+        "representatives (41)."],
+      "values": ["alkB.nwk",0.9,null,null,null,null,null,"alkB-pruned.nwk"]
+    },
+    {
+      "_": "== Examples of BLAST statistics and manipulation",
+      "task": "BlastTab.topHits_sorted.rb",
+      "description": ["Extract the best match of metagenome-derived proteins",
+        "(from the 'A metagenome') against a Gene Ontology collection."],
+      "values": ["sort","a_mg.cds-go.blast.tsv",null,null,null,null,1,null,null,
+        null,"a_mg.cds-go.blast-bm.tsv"]
+    },
+    {
+      "task": "BlastTab.sumPerHit.pl",
+      "description": ["Count the number of reads per gene in a mapping of a",
+        "metagenome to a metagenome-derived genes (from the 'A metagenome')."],
+      "values": [null,null,null,null,null,null,null,"a_mg.reads-cds.blast.tsv",
+        null,"a_mg.reads-cds.counts.tsv"]
+    },
+    {
+      "task": "BlastTab.sumPerHit.pl",
+      "description": ["Estimate the total abundance of Gene Ontology",
+        "annotations in the A metagenome, using metagenome-derived proteins,",
+        "and normalizing by the read counts of each protein."],
+      "values": ["a_mg.reads-cds.counts.tsv",null,null,null,null,true,null,
+        "a_mg.cds-go.blast.tsv",null,"a_mg.go.read-counts.tsv"]
+    },
+    {
+      "_": "== Examples of diversity ==",
+      "task": "Table.barplot.R",
+      "description": ["Barplot with the distribution of bacterial phyla in",
+        "four different sites, with taxa sorted by variance."],
+      "values": ["phyla_counts.tsv","250,100,75,200",null,null,null,null,null,
+        null,true,"var",2,null,null,"phyla_counts.pdf",10,null]
+    },
+    {
+      "task": "Chao1.pl",
+      "description": ["Phylum-richness estimated by the Chao1 index with 95%",
+        "confidence, using the distributions of bacterial phyla in four",
+        "different sites."],
+      "values": ["phyla_counts.tsv",null,1,null,null,true,null,
+         "phyla_chao1.tsv"]
+    },
+    {
+      "task": "AlphaDiversity.pl",
+      "description": ["Phylum-diversity estimated by the indices of Shannon",
+        "(H'), Inverse Simpson (1/Lambda), and true diversity of order 1 (1D),",
+        "using the distributions of bacterial phyla in four different sites."],
+      "values": ["phyla_counts.tsv",null,1,null,null,true,null,true,1,null,
+         "phyla_diversity.tsv"]
+    },
+    {
+      "_": "== Other miscelaneous examples ==",
+      "task": "CharTable.classify.rb",
+      "description": ["Classification of anthrax genomes based on can-SNPs, as",
+        "described in Van Ert 2007 (PLoS ONE 2(5):e461)."],
+      "values": ["anthrax-cansnp-data.tsv","anthrax-cansnp-key.tsv",
+        "anthrax-cansnp-classif.tsv","anthrax-cansnp-classif.nwk",null]
+    },
+    {
+      "task": "TRIBS.test.R",
+      "description": ["Test overclustering of Xanthomonas oryzae genomes",
+        "encoding for PilA using Transformed-space Resampling In Biased Sets",
+        "(TRIBS)."],
+      "values": ["Xanthomonas_oryzae.aai-mat.tsv","Xanthomonas_oryzae-PilA.txt",
+        5000,null,null,null,null,0,"Xanthomonas_oryzae-PilA.tribs.Rdata",100]
+    },
+    {
+      "task": "TRIBS.plot-test.R",
+      "description": ["Show the TRIBS-normalized distances between Xanthomonas",
+        "oryzae genomes (grey) and X. oryzae encoding for PilA (red)."],
+      "values": ["Xanthomonas_oryzae-PilA.tribs.Rdata",null,null,null,null,null,
+        null,null,"Xanthomonas_oryzae-PilA.tribs.pdf",null,null]
+    }
+  ]
+}

data/utils/enveomics/Manifest/tasks.json

@@ -0,0 +1,4 @@
+{
+  "_": "This file loads all the .json files inside 'Manifest/Tasks'.",
+  "_include": "Tasks/*.json"
+}

data/utils/enveomics/Pipelines/assembly.pbs/CONFIG.mock.bash

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+##################### VARIABLES
+# Queue: Preferred queue.  Delete (or comment) this line to allow
+# automatic detection:
+#QUEUE="biocluster-6"
+# If you set the QUEUE variable, you MUST set the WTIME variable
+# as well, containing the walltime to be asked for.  The WTIME
+# variable is ignored otherwise.
+WTIME="120:00:00"
+
+# Scratch:  This is where the output will be created.
+SCRATCH="$HOME/scratch/pipelines/assembly"
+
+# Data folder:  This is the folder that cointains the input files.
+DATA="$HOME/data/trim"
+
+# Location of Newbler's binaries
+BIN454="$HOME/454/bin"
+
+# Name(s) of the library(ies) to use, separated by spaces:
+# This is determined by the name of your input files.  For example,
+# if your input files are: LLSEP.CoupledReads.fa and LWP.CoupledReads.fa,
+# use:
+# LIBRARIES="LLSEP LWP"
+# It's strongly encouraged to use only one per CONFIG file.
+LIBRARIES="A";
+
+# Use .CoupledReads.fa and/or .SingleReads.fa (yes or no):
+USECOUPLED=yes
+USESINGLE=no
+
+# Insert length (in bp):  This is the average length of the entire insert,
+# not just the gap length.
+INSLEN=300
+
+# Number of CPUs to use (for SOAP and Newbler):
+PPN=16
+
+# RAM multiplier: Multiply the estimated required RAM by this number:
+RAMMULT=1
+
+# Maximum number of simultaneous jobs: Uncomment and increase these values if
+# you have increased resources (e.g., a dedicated queue); uncomment and decrease
+# if the resources are scarce (e.g., a very busy queue or other simultaneous jobs).
+#VELVETSIM=22
+#SOAPSIM=8
+
+# Extra parameters for Velvet: Any additional parameters to be passed to
+# velvetg or velveth.  If you have MP data, consider adding the option
+# -shortMatePaired yes to VELVETG_EXTRA.  If you have Nextera, consider
+# adding the option above, plus the option -ins_length_sd <integer>, to
+# indicate the standard deviation of the insert size.  By default, the
+# SD is assumed to be 10% of the average, but Nextera produces much
+# wider distribution of sizes (i.e., larger SD).  Typically you shouldn't
+# need to add anything in VELVETH_EXTRA.
+VELVETH_EXTRA=""
+VELVETG_EXTRA=""
+
+# Clean non-essential files (yes or no):
+CLEANUP=yes
+
+# Best k-mers:  Space-delimited list of kmers selected from Velvet and SOAP.
+# This is to be modified at the begining of step 4, and it's ignored in all
+# the other steps.
+K_VELVET="21 23 35"
+K_SOAP="21 23 35"
+
+

data/utils/enveomics/Pipelines/assembly.pbs/FastA.N50.pl

@@ -0,0 +1 @@
+../../Scripts/FastA.N50.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.filterN.pl

@@ -0,0 +1 @@
+../../Scripts/FastA.filterN.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.length.pl

@@ -0,0 +1 @@
+../../Scripts/FastA.length.pl

data/utils/enveomics/Pipelines/assembly.pbs/README.md

@@ -0,0 +1,189 @@
+@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
+
+@update: Mar-17-2013
+
+@license: artistic 2.0
+
+@status: semi
+
+@pbs: yes
+
+# IMPORTANT
+
+This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
+are free to use it in other platforms with adequate adjustments.  It is largely
+based on Luo _et al._ 2012, ISME J.
+
+# PURPOSE
+
+This pipeline assemblies coupled and/or single reads from one or more libraries.
+It assumes that the reads have been quality-checked and trimmed.
+
+# HELP
+
+1. Files preparation:
+
+   1.1. Copy this folder to the cluster.
+   
+   1.2. Copy the sequences to the cluster.  Only trimmed/filtered reads are used.
+      All the files are expected to be in the same folder, and the filenames must
+      end in `.CoupledReads.fa` or `.SingleReads.fa`.
+   
+   1.3. Copy the file `CONFIG.mock.bash` to `CONFIG.<name>.bash`, where `<name>` is a
+      short name for your run (avoid characters other than alphanumeric).
+   
+   1.4. Change the variables in `CONFIG.<name>.bash`.  Notice that this pipeline
+      supports running several libraries at the same time, but it's strongly
+      recomended to run only one per config file, because the insert length
+      (in step 2) and the selected k-mers (in step 3) are fixed for all the
+      included libraries.  Also, there is a technical consideration:  The first
+      step will execute parallel jobs for each odd number between 21 and 63, and
+      SOAP will use 16 CPUs by default, which means 357 CPUs will be requested
+      per library in step 2.  It's a bad idea to run many libraries at the same
+      time.
+
+   1.5. If you have Mate-paired datasets (for example, prepared with Nextera), first
+      reverse-complement all the reads.  See also the `VELVETG_EXTRA` variable in
+      the `CONFIG.<name>.bash` file.
+
+2. Velvet and SOAP assembly:
+   
+   2.1. Execute `./RUNME-2.bash <name>` in the head node (see [troubleshooting](#troubleshooting) #1).
+   
+   2.2. Monitor the tasks named velvet_* and soap_*.
+   
+   2.3. Once completed, make sure the files .proc contain only the
+      word "done".  To do this, you may execute:
+```
+      grep -v '^done$' *.proc
+```
+
+      If successful, the output of the above command should be empty.  See
+      [Troubleshooting](#troubleshooting) #2 and #3 below if one or more of your jobs failed.
+
+3. K-mers selection:
+   
+   3.1. If you completed step 2, execute `./RUNME-3.bash <name>` in the head
+      node.
+   
+   3.2. Once completed, download and open the files `*.n50.pdf`.
+   
+   3.3. Select the three "best" k-mers for Velvet and for SOAP (they don't
+      have to be the same).  There is no well-tested method to select the
+      "best", and this is why this protocol is not automated, but semi-
+      automated.  A generally good rule-of-thumb is: pick one that optimizes
+      the amount of sequences used (these are the grey bars in the plot;
+      usually this is the smallest k-mer), pick one that optimizes the N50
+      (this is the dashed red line; usually this is a large k-mer), and pick
+      one that optimizes both (something in the middle).  You can select
+      more or less than three k-mers, this is just a suggestion.
+
+4. Newbler assembly:
+   
+   4.1. Edit the file `CONFIG.<name>.bash`: set the variables `K_VELVET` and
+      `K_SOAP` to contain the lists of "best" selected k-mers for Velvet and
+      SOAP, respectively.
+   
+   4.2. Execute `./RUNME-4.bash <name>` in the head node.
+   
+   4.3. Monitor the task newbler_*.  Once finished, your assembly is ready.
+      Once completed, make sure the file .newbler.proc contain only the
+      word "done".  To do this, you may execute:
+```
+      grep -v '^done$' *.proc
+```
+      If successful, the output should be empty.
+   
+   4.4. The final assembly should be located in the `SCRATCH` path, in a folder
+      named `<lib>.newbler/assembly/`.  The file `454AllContigs.fna` contains
+      all the assembled contigs, `454LargeContigs.fna` contains the contigs
+      with 500bp or more in length, and `454NewblerMetrics.txt` contains some
+      relevant statistics.
+
+
+# Comments
+
+* Some scripts contained in this package are actually symlinks to files in the
+  _Scripts_ folder.  Check the existance of these files when copied to
+  the cluster.
+
+# Troubleshooting
+
+1. Do I really have to change directory (`cd`) to the pipeline's folder everytime
+   I want to execute something?
+   
+   No.  Not really.  For simplicity, this file tells you to execute, for example,
+   `./RUNME-2.bash`.  However, you don't really have to be there, you can execute it
+   from any location.  For example, if you saved this pipeline in your home
+   directory, you can just execute `~/assembly.pbs/RUNME-2.bash` insted from any
+   location in the head node.
+
+2. I executed step 2, and Velvet worked but SOAP failed (or vice versa).  Can I
+   submit only one of them?
+
+   Yes.  To execute only Velvet, run:
+```
+   ./RUNME-2.bash <name> velvet
+```
+
+   To execute only SOAP, run:
+```
+   ./RUNME-2.bash <name> soap
+```
+
+3. I ran step 2, and most of the jobs finished, but few of them failed.  Can I
+   submit only few K-mers?
+
+   Yes.  To execute one kmer (say, the k-mer 33 of SOAP), run:
+```
+   ./RUNME-2.bash <name> soap 33
+```
+
+   You can also execute more than one kmer, using a comma-separated list.  For
+   example, to re-submit the k-mers 37, 39, and 41 of Velvet, run:
+```
+   ./RUNME-2.bash <name> velvet 37,39,41
+```
+
+4. What are the numbers on the job names of step 2?
+
+   The K-mer.  Each k-mer has it's own job, but they are "arrayed", to simplify
+   administration: notice that all the jobs of Velvet and all the jobs of SOAP
+   share the same job ID.
+
+5. Some jobs are being killed, why?
+
+   5.1. First, check the log file created by the pipeline.  The name is typically
+      the output prefix and the .log extension.  For velvet, there are two log files,
+      the `.glog` and the `.hlog`.  You may find the problem there.
+
+   5.2. Now, check the error file in your HOME directory.  The name depends on the
+      job, the library and the task.  For example: `~/soap_Mg_2-37.e1999838` is the
+      error file for step 2, task soap, library Mg_2, k-mer 37.  The appending
+      number after the 'e' is the job ID.  If this file contains errors probably
+      related to the pipeline, please let me know.
+
+   5.3. If you still have no clues, check the output file in your `HOME` directory.  The
+      name is just like the name of the error file (see #5.2 above), but with 'o'
+      instead of 'e'.  Compare the lines 'Resources' (what we asked the scheduler for)
+      and 'Rsrc Used' (what the job actually used).  A typical problem is that your
+      job may need more RAM than we asked for (the value of 'mem' in both lines).  If
+      the RAM used is larger than the RAM requested, the scheduler probably killed
+      your job.  To solve this, just go to your config file, and set the variable
+      RAMMULT to a number larger than 1.  For example, if you want to ask for double the
+      RAM, set `RAMMULT=2`.  You can also include simple arithmetic operations, like
+      `RAMMULT=3/2`.  If you want to add a fixed ammount of RAM, in Gib, use addition.
+      For example, to add 10G, set `RAMMULT=1+10`.
+
+   5.4.  Still no idea?  Try running the job again, sometimes the jobs fail with no
+      apparent reason, but they succeed when re-submited.  If your job keeps failing,
+      please gather as much information (the log, error and output files should be
+      enough) and let me take a look.
+
+6. In the step 2, some k-mers keep failing, and I just want to give up on them, can I?
+   
+   Yes.  Step 3 will analyze only completed jobs, so you can just ignore these faulty
+   k-mers.  Very small k-mers, for example, sometimes need too much memory, and very
+   large k-mers in Velvet sometimes need too much time.  If you don't think you're
+   missing too much, just ignore them.
+