miga-base 0.7.24.0 → 0.7.26.0

data/utils/enveomics/Manifest/categories.json

@@ -1,156 +0,0 @@
-{
-  "categories": {
-    "Sequence similarity search": {
-      "Statistics": [
-        "BedGraph.tad.rb",
-        "BedGraph.window.rb",
-        "BlastPairwise.AAsubs.pl",
-        "BlastTab.advance.bash",
-        "BlastTab.recplot2.R",
-        "BlastTab.seqdepth.pl",
-        "BlastTab.seqdepth_nomedian.pl",
-        "BlastTab.seqdepth_ZIP.pl",
-        "BlastTab.sumPerHit.pl",
-        "FastQ.test-error.rb",
-        "RecPlot2.compareIdentities.R"
-      ],
-      "Manipulation": [
-        "BlastTab.addlen.rb",
-        "BlastTab.best_hit_sorted.pl",
-        "BlastTab.catsbj.pl",
-        "BlastTab.cogCat.rb",
-        "BlastTab.filter.pl",
-        "BlastTab.kegg_pep2path_rest.pl",
-        "BlastTab.pairedHits.rb",
-        "BlastTab.subsample.pl",
-        "BlastTab.taxid2taxrank.pl",
-        "BlastTab.topHits_sorted.rb"
-      ],
-      "Execution": [
-        "aai.rb",
-        "ani.rb",
-        "HMM.haai.rb",
-        "rbm.rb"
-      ]
-    },
-    "Sequence analyses": {
-      "Statistics": [
-        "FastA.gc.pl",
-        "FastA.length.pl",
-        "FastA.N50.pl",
-        "FastA.qlen.pl",
-        "FastQ.test-error.rb"
-      ],
-      "Manipulation": [
-        "FastA.extract.rb",
-        "FastA.filter.pl",
-        "FastA.filterLen.pl",
-        "FastA.filterN.pl",
-        "FastA.fragment.rb",
-        "FastA.interpose.pl",
-        "FastA.mask.rb",
-        "FastA.per_file.pl",
-        "FastA.rename.pl",
-        "FastA.revcom.pl",
-        "FastA.sample.rb",
-        "FastA.slider.pl",
-        "FastA.split.pl",
-        "FastA.split.rb",
-        "FastA.subsample.pl",
-        "FastA.tag.rb",
-        "FastA.wrap.rb",
-        "FastQ.filter.pl",
-        "FastQ.interpose.pl",
-        "FastQ.offset.pl",
-        "FastQ.split.pl",
-        "FastQ.tag.rb",
-        "FastQ.toFastA.awk"
-      ]
-    },
-    "Diversity": {
-      "Community": [
-        "AlphaDiversity.pl",
-        "Chao1.pl",
-        "Table.barplot.R"
-      ],
-      "Population": [
-        "VCF.SNPs.rb",
-        "VCF.KaKs.rb"
-      ]
-    },
-    "Annotation": {
-      "Database mapping": [
-        "BlastTab.kegg_pep2path_rest.pl",
-        "BlastTab.taxid2taxrank.pl",
-        "EBIseq2tax.rb",
-        "NCBIacc2tax.rb",
-        "gi2tax.rb",
-        "M5nr.getSequences.rb",
-        "RefSeq.download.bash",
-        "SRA.download.bash"
-      ],
-      "Tables": [
-        "Table.barplot.R",
-        "GenBank.add_fields.rb",
-        "MyTaxa.fragsByTax.pl",
-        "Table.df2dist.R",
-        "Table.filter.pl",
-        "Table.merge.pl",
-        "Table.replace.rb",
-        "Table.round.rb",
-        "Table.split.pl"
-      ],
-      "Search": [
-        "HMM.essential.rb",
-        "HMM.haai.rb",
-        "HMMsearch.extractIds.rb",
-        "ogs.annotate.rb",
-        "ogs.core-pan.rb",
-        "ogs.extract.rb",
-        "ogs.mcl.rb",
-        "ogs.stats.rb",
-        "ogs.rb"
-      ]
-    },
-    "Other data": {
-      "Phylogenetic and other distances": [
-        "CharTable.classify.rb",
-        "JPlace.distances.rb",
-        "JPlace.to_iToL.rb",
-        "Newick.autoprune.R",
-        "TRIBS.test.R",
-        "TRIBS.plot-test.R",
-        "Table.df2dist.R"
-      ],
-      "Taxonomic": [
-        "CharTable.classify.rb",
-        "EBIseq2tax.rb",
-        "NCBIacc2tax.rb",
-        "Table.barplot.R",
-        "gi2tax.rb",
-        "MyTaxa.fragsByTax.pl",
-        "MyTaxa.seq-taxrank.rb",
-        "Taxonomy.silva2ncbi.rb"
-      ],
-      "Alignments": [
-        "AAsubs.log2ratio.rb",
-        "Aln.cat.rb",
-        "Aln.convert.pl",
-        "BlastPairwise.AAsubs.pl"
-      ],
-      "Clustering": [
-        "ogs.mcl.rb",
-        "clust.rand.rb"
-      ],
-      "Read recruitments": [
-        "BedGraph.tad.rb",
-        "BedGraph.window.rb",
-        "BlastTab.catsbj.pl",
-        "BlastTab.pairedHits.rb",
-        "BlastTab.recplot2.R",
-        "GFF.catsbj.pl",
-        "RecPlot2.compareIdentities.R"
-      ]
-    }
-  }
-}

data/utils/enveomics/Manifest/examples.json

@@ -1,154 +0,0 @@
-{
-  "_": "Input files and directories are included in the 'Tests' folder.",
-  "examples": [
-    {
-      "_": "== Examples of genome comparisons ==",
-      "task": "ogs.stats.rb",
-      "description": ["Statistics on the groups of orthology in the Primate",
-        "Lentivirus Group, including HIV-1, HIV-2, and SIV."],
-      "values": ["primate_lentivirus.ogs",null,null,null,null,null]
-    },
-    {
-      "task": "ani.rb",
-      "description": ["Average Nucleotide Identity (ANI) between two strains",
-        "of Mycoplasma genitalium (M2288 and M2321)."],
-      "values": ["Mgen_M2288.fna","Mgen_M2321.fna",null,null,null,null,null,
-        null,null,null,null,null,null,null,null,null,null,null,null,null,null,
-        null,null,null]
-    },
-    {
-      "task": "aai.rb",
-      "description": ["Average Amino acid Identity (AAI) between Mycoplasma",
-        "genitalium (Bacteria) and Nanoarchaeum equitans (Archaea)."],
-      "values": ["Mgen_M2288.faa","Nequ_Kin4M.faa",null,null,null,null,null,
-        null,null,null,null,null,null,null,null,null,null,null,null,null,null,
-        null,null,null]
-    },
-    {
-      "task": "rbm.rb",
-      "description": ["Reciprocal Best Matches between the proteomes of the",
-        "two major HIV types (HIV-1 and HIV-2)."],
-      "values": ["hiv1.faa","hiv2.faa",null,null,null,null,null,null,null,null,
-        null,null,"hiv1-hiv2.rbm"]
-    },
-    {
-      "task": "ogs.mcl.rb",
-      "description": ["Groups of orthology in the Primate Letivirus Group,",
-         "including HIV-1, HIV-2, and SIV."],
-      "values": ["primate_lentivirus.ogs","primate_lentivirus.rbm",null,null,
-        null,null,null,null,null,null,null,null]
-    },
-    {
-      "task": "Table.df2dist.R",
-      "description": ["Transforms a list of AAI values between Xanthomonas",
-        "oryzae genomes into a distance matrix."],
-      "values": ["Xanthomonas_oryzae.aai.tsv",null,null,null,null,100.0,
-        "Xanthomonas_oryzae.aai-mat.tsv"]
-    },
-    {
-      "_": "== Recruitment plots",
-      "task": "BlastTab.catsbj.pl",
-      "description": ["Prepares recruitment plot files for a comparison",
-        "between a virome containing HIV and the HIV-1 genome."],
-      "values": [null,null,null,null,"hiv1.fna","hiv_mix-hiv1.blast.tsv"]
-    },
-    {
-      "task": "BlastTab.recplot2.R",
-      "description": ["Generates recruitment plots for a comparison",
-        "between a virome containing HIV and the HIV-1 genome."],
-      "values": ["hiv_mix-hiv1.blast.tsv",50,100,null,null,null,null,null,null,
-        null,null,null,"hiv_mix-hiv1.Rdata","hiv_mix-hiv1.pdf",null,null]
-    },
-    {
-      "_": "== Examples of functional annotations ==",
-      "task": "HMM.essential.rb",
-      "description": ["Typical single-copy bacterial genes present in",
-        "Mycoplasma genitalium."],
-      "values": ["Mgen_M2288.faa",null,null,null,null,null,null,true,null,null,
-        null,null,null,null,null,null,null,null,null]
-    },
-    {
-      "task": "HMM.essential.rb",
-      "description": ["Typical single-copy archaeal genes present in",
-        "Nanoarchaeum equitans."],
-      "values": ["Mgen_M2288.faa",null,null,null,null,null,null,null,true,null,
-        null,null,null,null,null,null,null,null,null]
-    },
-    {
-      "task": "Newick.autoprune.R",
-      "description": ["Prune an AlkB tree with 110 tips to get only distant",
-        "representatives (41)."],
-      "values": ["alkB.nwk",0.9,null,null,null,null,null,"alkB-pruned.nwk"]
-    },
-    {
-      "_": "== Examples of BLAST statistics and manipulation",
-      "task": "BlastTab.topHits_sorted.rb",
-      "description": ["Extract the best match of metagenome-derived proteins",
-        "(from the 'A metagenome') against a Gene Ontology collection."],
-      "values": ["sort","a_mg.cds-go.blast.tsv",null,null,null,null,1,null,null,
-        null,"a_mg.cds-go.blast-bm.tsv"]
-    },
-    {
-      "task": "BlastTab.sumPerHit.pl",
-      "description": ["Count the number of reads per gene in a mapping of a",
-        "metagenome to a metagenome-derived genes (from the 'A metagenome')."],
-      "values": [null,null,null,null,null,null,null,"a_mg.reads-cds.blast.tsv",
-        null,"a_mg.reads-cds.counts.tsv"]
-    },
-    {
-      "task": "BlastTab.sumPerHit.pl",
-      "description": ["Estimate the total abundance of Gene Ontology",
-        "annotations in the A metagenome, using metagenome-derived proteins,",
-        "and normalizing by the read counts of each protein."],
-      "values": ["a_mg.reads-cds.counts.tsv",null,null,null,null,true,null,
-        "a_mg.cds-go.blast.tsv",null,"a_mg.go.read-counts.tsv"]
-    },
-    {
-      "_": "== Examples of diversity ==",
-      "task": "Table.barplot.R",
-      "description": ["Barplot with the distribution of bacterial phyla in",
-        "four different sites, with taxa sorted by variance."],
-      "values": ["phyla_counts.tsv","250,100,75,200",null,null,null,null,null,
-        null,true,"var",2,null,null,"phyla_counts.pdf",10,null]
-    },
-    {
-      "task": "Chao1.pl",
-      "description": ["Phylum-richness estimated by the Chao1 index with 95%",
-        "confidence, using the distributions of bacterial phyla in four",
-        "different sites."],
-      "values": ["phyla_counts.tsv",null,1,null,null,true,null,
-         "phyla_chao1.tsv"]
-    },
-    {
-      "task": "AlphaDiversity.pl",
-      "description": ["Phylum-diversity estimated by the indices of Shannon",
-        "(H'), Inverse Simpson (1/Lambda), and true diversity of order 1 (1D),",
-        "using the distributions of bacterial phyla in four different sites."],
-      "values": ["phyla_counts.tsv",null,1,null,null,true,null,true,1,null,
-         "phyla_diversity.tsv"]
-    },
-    {
-      "_": "== Other miscelaneous examples ==",
-      "task": "CharTable.classify.rb",
-      "description": ["Classification of anthrax genomes based on can-SNPs, as",
-        "described in Van Ert 2007 (PLoS ONE 2(5):e461)."],
-      "values": ["anthrax-cansnp-data.tsv","anthrax-cansnp-key.tsv",
-        "anthrax-cansnp-classif.tsv","anthrax-cansnp-classif.nwk",null]
-    },
-    {
-      "task": "TRIBS.test.R",
-      "description": ["Test overclustering of Xanthomonas oryzae genomes",
-        "encoding for PilA using Transformed-space Resampling In Biased Sets",
-        "(TRIBS)."],
-      "values": ["Xanthomonas_oryzae.aai-mat.tsv","Xanthomonas_oryzae-PilA.txt",
-        5000,null,null,null,null,0,"Xanthomonas_oryzae-PilA.tribs.Rdata",100]
-    },
-    {
-      "task": "TRIBS.plot-test.R",
-      "description": ["Show the TRIBS-normalized distances between Xanthomonas",
-        "oryzae genomes (grey) and X. oryzae encoding for PilA (red)."],
-      "values": ["Xanthomonas_oryzae-PilA.tribs.Rdata",null,null,null,null,null,
-        null,null,"Xanthomonas_oryzae-PilA.tribs.pdf",null,null]
-    }
-  ]
-}

data/utils/enveomics/Manifest/tasks.json

@@ -1,4 +0,0 @@
-{
-  "_": "This file loads all the .json files inside 'Manifest/Tasks'.",
-  "_include": "Tasks/*.json"
-}

data/utils/enveomics/Pipelines/assembly.pbs/CONFIG.mock.bash

@@ -1,69 +0,0 @@
-#!/bin/bash
-
-##################### VARIABLES
-# Queue: Preferred queue.  Delete (or comment) this line to allow
-# automatic detection:
-#QUEUE="biocluster-6"
-# If you set the QUEUE variable, you MUST set the WTIME variable
-# as well, containing the walltime to be asked for.  The WTIME
-# variable is ignored otherwise.
-WTIME="120:00:00"
-
-# Scratch:  This is where the output will be created.
-SCRATCH="$HOME/scratch/pipelines/assembly"
-
-# Data folder:  This is the folder that cointains the input files.
-DATA="$HOME/data/trim"
-
-# Location of Newbler's binaries
-BIN454="$HOME/454/bin"
-
-# Name(s) of the library(ies) to use, separated by spaces:
-# This is determined by the name of your input files.  For example,
-# if your input files are: LLSEP.CoupledReads.fa and LWP.CoupledReads.fa,
-# use:
-# LIBRARIES="LLSEP LWP"
-# It's strongly encouraged to use only one per CONFIG file.
-LIBRARIES="A";
-
-# Use .CoupledReads.fa and/or .SingleReads.fa (yes or no):
-USECOUPLED=yes
-USESINGLE=no
-
-# Insert length (in bp):  This is the average length of the entire insert,
-# not just the gap length.
-INSLEN=300
-
-# Number of CPUs to use (for SOAP and Newbler):
-PPN=16
-
-# RAM multiplier: Multiply the estimated required RAM by this number:
-RAMMULT=1
-
-# Maximum number of simultaneous jobs: Uncomment and increase these values if
-# you have increased resources (e.g., a dedicated queue); uncomment and decrease
-# if the resources are scarce (e.g., a very busy queue or other simultaneous jobs).
-#VELVETSIM=22
-#SOAPSIM=8
-
-# Extra parameters for Velvet: Any additional parameters to be passed to
-# velvetg or velveth.  If you have MP data, consider adding the option
-# -shortMatePaired yes to VELVETG_EXTRA.  If you have Nextera, consider
-# adding the option above, plus the option -ins_length_sd <integer>, to
-# indicate the standard deviation of the insert size.  By default, the
-# SD is assumed to be 10% of the average, but Nextera produces much
-# wider distribution of sizes (i.e., larger SD).  Typically you shouldn't
-# need to add anything in VELVETH_EXTRA.
-VELVETH_EXTRA=""
-VELVETG_EXTRA=""
-
-# Clean non-essential files (yes or no):
-CLEANUP=yes
-
-# Best k-mers:  Space-delimited list of kmers selected from Velvet and SOAP.
-# This is to be modified at the begining of step 4, and it's ignored in all
-# the other steps.
-K_VELVET="21 23 35"
-K_SOAP="21 23 35"
-
-

data/utils/enveomics/Pipelines/assembly.pbs/FastA.N50.pl

@@ -1 +0,0 @@
-../../Scripts/FastA.N50.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.filterN.pl

@@ -1 +0,0 @@
-../../Scripts/FastA.filterN.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.length.pl

@@ -1 +0,0 @@
-../../Scripts/FastA.length.pl

data/utils/enveomics/Pipelines/assembly.pbs/README.md

@@ -1,189 +0,0 @@
-@author: Luis Miguel Rodriguez-R <lmrodriguezr at gmail dot com>
-
-@update: Mar-17-2013
-
-@license: artistic 2.0
-
-@status: semi
-
-@pbs: yes
-
-# IMPORTANT
-
-This pipeline was developed for the [PACE cluster](http://pace.gatech.edu/).  You
-are free to use it in other platforms with adequate adjustments.  It is largely
-based on Luo _et al._ 2012, ISME J.
-
-# PURPOSE
-
-This pipeline assemblies coupled and/or single reads from one or more libraries.
-It assumes that the reads have been quality-checked and trimmed.
-
-# HELP
-
-1. Files preparation:
-
-   1.1. Copy this folder to the cluster.
-   
-   1.2. Copy the sequences to the cluster.  Only trimmed/filtered reads are used.
-      All the files are expected to be in the same folder, and the filenames must
-      end in `.CoupledReads.fa` or `.SingleReads.fa`.
-   
-   1.3. Copy the file `CONFIG.mock.bash` to `CONFIG.<name>.bash`, where `<name>` is a
-      short name for your run (avoid characters other than alphanumeric).
-   
-   1.4. Change the variables in `CONFIG.<name>.bash`.  Notice that this pipeline
-      supports running several libraries at the same time, but it's strongly
-      recomended to run only one per config file, because the insert length
-      (in step 2) and the selected k-mers (in step 3) are fixed for all the
-      included libraries.  Also, there is a technical consideration:  The first
-      step will execute parallel jobs for each odd number between 21 and 63, and
-      SOAP will use 16 CPUs by default, which means 357 CPUs will be requested
-      per library in step 2.  It's a bad idea to run many libraries at the same
-      time.
-
-   1.5. If you have Mate-paired datasets (for example, prepared with Nextera), first
-      reverse-complement all the reads.  See also the `VELVETG_EXTRA` variable in
-      the `CONFIG.<name>.bash` file.
-
-2. Velvet and SOAP assembly:
-   
-   2.1. Execute `./RUNME-2.bash <name>` in the head node (see [troubleshooting](#troubleshooting) #1).
-   
-   2.2. Monitor the tasks named velvet_* and soap_*.
-   
-   2.3. Once completed, make sure the files .proc contain only the
-      word "done".  To do this, you may execute:
-```
-      grep -v '^done$' *.proc
-```
-
-      If successful, the output of the above command should be empty.  See
-      [Troubleshooting](#troubleshooting) #2 and #3 below if one or more of your jobs failed.
-
-3. K-mers selection:
-   
-   3.1. If you completed step 2, execute `./RUNME-3.bash <name>` in the head
-      node.
-   
-   3.2. Once completed, download and open the files `*.n50.pdf`.
-   
-   3.3. Select the three "best" k-mers for Velvet and for SOAP (they don't
-      have to be the same).  There is no well-tested method to select the
-      "best", and this is why this protocol is not automated, but semi-
-      automated.  A generally good rule-of-thumb is: pick one that optimizes
-      the amount of sequences used (these are the grey bars in the plot;
-      usually this is the smallest k-mer), pick one that optimizes the N50
-      (this is the dashed red line; usually this is a large k-mer), and pick
-      one that optimizes both (something in the middle).  You can select
-      more or less than three k-mers, this is just a suggestion.
-
-4. Newbler assembly:
-   
-   4.1. Edit the file `CONFIG.<name>.bash`: set the variables `K_VELVET` and
-      `K_SOAP` to contain the lists of "best" selected k-mers for Velvet and
-      SOAP, respectively.
-   
-   4.2. Execute `./RUNME-4.bash <name>` in the head node.
-   
-   4.3. Monitor the task newbler_*.  Once finished, your assembly is ready.
-      Once completed, make sure the file .newbler.proc contain only the
-      word "done".  To do this, you may execute:
-```
-      grep -v '^done$' *.proc
-```
-      If successful, the output should be empty.
-   
-   4.4. The final assembly should be located in the `SCRATCH` path, in a folder
-      named `<lib>.newbler/assembly/`.  The file `454AllContigs.fna` contains
-      all the assembled contigs, `454LargeContigs.fna` contains the contigs
-      with 500bp or more in length, and `454NewblerMetrics.txt` contains some
-      relevant statistics.
-
-
-# Comments
-
-* Some scripts contained in this package are actually symlinks to files in the
-  _Scripts_ folder.  Check the existance of these files when copied to
-  the cluster.
-
-# Troubleshooting
-
-1. Do I really have to change directory (`cd`) to the pipeline's folder everytime
-   I want to execute something?
-   
-   No.  Not really.  For simplicity, this file tells you to execute, for example,
-   `./RUNME-2.bash`.  However, you don't really have to be there, you can execute it
-   from any location.  For example, if you saved this pipeline in your home
-   directory, you can just execute `~/assembly.pbs/RUNME-2.bash` insted from any
-   location in the head node.
-
-2. I executed step 2, and Velvet worked but SOAP failed (or vice versa).  Can I
-   submit only one of them?
-
-   Yes.  To execute only Velvet, run:
-```
-   ./RUNME-2.bash <name> velvet
-```
-
-   To execute only SOAP, run:
-```
-   ./RUNME-2.bash <name> soap
-```
-
-3. I ran step 2, and most of the jobs finished, but few of them failed.  Can I
-   submit only few K-mers?
-
-   Yes.  To execute one kmer (say, the k-mer 33 of SOAP), run:
-```
-   ./RUNME-2.bash <name> soap 33
-```
-
-   You can also execute more than one kmer, using a comma-separated list.  For
-   example, to re-submit the k-mers 37, 39, and 41 of Velvet, run:
-```
-   ./RUNME-2.bash <name> velvet 37,39,41
-```
-
-4. What are the numbers on the job names of step 2?
-
-   The K-mer.  Each k-mer has it's own job, but they are "arrayed", to simplify
-   administration: notice that all the jobs of Velvet and all the jobs of SOAP
-   share the same job ID.
-
-5. Some jobs are being killed, why?
-
-   5.1. First, check the log file created by the pipeline.  The name is typically
-      the output prefix and the .log extension.  For velvet, there are two log files,
-      the `.glog` and the `.hlog`.  You may find the problem there.
-
-   5.2. Now, check the error file in your HOME directory.  The name depends on the
-      job, the library and the task.  For example: `~/soap_Mg_2-37.e1999838` is the
-      error file for step 2, task soap, library Mg_2, k-mer 37.  The appending
-      number after the 'e' is the job ID.  If this file contains errors probably
-      related to the pipeline, please let me know.
-
-   5.3. If you still have no clues, check the output file in your `HOME` directory.  The
-      name is just like the name of the error file (see #5.2 above), but with 'o'
-      instead of 'e'.  Compare the lines 'Resources' (what we asked the scheduler for)
-      and 'Rsrc Used' (what the job actually used).  A typical problem is that your
-      job may need more RAM than we asked for (the value of 'mem' in both lines).  If
-      the RAM used is larger than the RAM requested, the scheduler probably killed
-      your job.  To solve this, just go to your config file, and set the variable
-      RAMMULT to a number larger than 1.  For example, if you want to ask for double the
-      RAM, set `RAMMULT=2`.  You can also include simple arithmetic operations, like
-      `RAMMULT=3/2`.  If you want to add a fixed ammount of RAM, in Gib, use addition.
-      For example, to add 10G, set `RAMMULT=1+10`.
-
-   5.4.  Still no idea?  Try running the job again, sometimes the jobs fail with no
-      apparent reason, but they succeed when re-submited.  If your job keeps failing,
-      please gather as much information (the log, error and output files should be
-      enough) and let me take a look.
-
-6. In the step 2, some k-mers keep failing, and I just want to give up on them, can I?
-   
-   Yes.  Step 3 will analyze only completed jobs, so you can just ignore these faulty
-   k-mers.  Very small k-mers, for example, sometimes need too much memory, and very
-   large k-mers in Velvet sometimes need too much time.  If you don't think you're
-   missing too much, just ignore them.
-