miga-base 0.3.1.7 → 0.3.2.0

data/utils/distance/database.rb

@@ -0,0 +1,86 @@
+
+require 'sqlite3'
+
+module MiGA::DistanceRunner::Database
+  # Check for corrupt files and create empty databases
+  def initialize_dbs!(for_ref)
+    @dbs = {}
+    @tmp_dbs = {}
+    @db_counts = {}
+    {haai: :aai, aai: :aai, ani: :ani}.each do |m, t|
+      @db_counts[m] = 0
+      @dbs[m] = for_ref ? ref_db(m) : query_db(m)
+      # Remove if corrupt
+      if File.size?(dbs[m])
+        begin
+          SQLite3::Database.new(dbs[m]) do |conn|
+            conn.execute "select count(*) from #{t};"
+          end
+        rescue SQLite3::SQLException
+          FileUtils.rm dbs[m]
+        end
+      end
+      # Initialize if it doesn't exist
+      SQLite3::Database.new(dbs[m]) do |conn|
+        conn.execute "create table if not exists #{t}(" +
+            "seq1 varchar(256), seq2 varchar(256), " +
+            "#{t} float, sd float, n int, omega int" +
+          ")"
+      end unless File.size? dbs[m]
+      # Copy over to (local) temporals
+      @tmp_dbs[m] = tmp_file("#{m}.db")
+      FileUtils.cp(dbs[m], tmp_dbs[m])
+    end
+  end
+
+  # Path to the database +metric+ for +dataset_name+ in +project+
+  # (assumes that +dataset_name+ is a reference dataset)
+  def ref_db(metric, dataset_name=nil)
+    dataset_name ||= dataset.name
+    b = case metric
+    when :haai
+      "01.haai/#{dataset_name}.db"
+    when :aai
+      "02.aai/#{dataset_name}.db"
+    when :ani
+      "03.ani/#{dataset_name}.db"
+    end
+    File.expand_path(b, home)
+  end
+
+  # Path to the database +metric+ for +dataset+ (assumes that +dataset+ is a
+  # query dataset)
+  def query_db(metric)
+    File.expand_path("#{dataset.name}.#{metric}.db", home)
+  end
+
+  # Get the stored +metric+ value against +target+
+  def stored_value(target, metric)
+    # Check if self.dataset -> target is done (previous run)
+    y = value_from_db(dataset.name, target.name, tmp_dbs[metric], metric)
+    return y unless y.nil? or y.zero?
+    # Check if self.dataset <- target is done (another thread)
+    if dataset.is_ref? and project.path==ref_project.path
+      y = value_from_db(target.name, dataset.name, ref_db(metric, target.name), metric)
+      return y unless y.nil? or y.zero?
+    end
+    nil
+  end
+
+  # Get the value of +metric+ in the +db+ database between +n1+ and +n2+
+  def value_from_db(n1, n2, db, metric)
+    y = nil
+    SQLite3::Database.new(db) do |conn|
+      y = conn.execute("select #{metric} from #{metric} where seq1=? and seq2=?", [n1, n2]).first
+      y = y.first unless y.nil?
+    end if File.size? db
+    y
+  end
+
+  # Iterates for each entry in +db+
+  def foreach_in_db(db, metric, &blk)
+    SQLite3::Database.new(db) do |conn|
+      conn.execute("select * from #{metric}").each{ |r| blk[r] }
+    end
+  end
+end

data/utils/distance/pipeline.rb

@@ -0,0 +1,98 @@
+
+# High-end pipelines for DistanceRunner
+module MiGA::DistanceRunner::Pipeline
+
+  # Recursively classify the dataset, returning an Array with two entries:
+  # classification and cluster number
+  def classify(clades, classif, metric, result_fh, val_cls=nil)
+    dir = File.expand_path(classif, clades)
+    med = File.expand_path("miga-project.medoids", dir)
+    return [classif,val_cls] unless File.size? med
+    max_val = 0
+    val_med = ""
+    val_cls = nil
+    i_n = 0
+    File.open(med, "r") do |med_fh|
+      med_fh.each_line do |med_ln|
+        i_n += 1
+        med_ln.chomp!
+        val = send(metric, ref_project.dataset(med_ln))
+        if !val.nil? and val >= max_val
+          max_val = val
+          val_med = med_ln
+          val_cls = i_n
+          puts "[#{classif}] New max: #{val_med} (#{val_cls}): #{max_val}"
+        end
+      end
+    end
+    classif = File.expand_path("miga-project.sc-#{val_cls}", classif)
+    result_fh.puts [val_cls, val_med, max_val, classif].join("\t")
+    classify(clades, classif, metric, result_fh, val_cls)
+  end
+
+  # Builds a tree with all visited medoids from any classification level
+  def build_medoids_tree(metric)
+    db = query_db(metric)
+    return unless File.size? db
+    out_base = File.expand_path(dataset.name, home)
+    ds_matrix = "#{out_base}.txt"
+    ds_matrix_fh = File.open(ds_matrix, "w")
+    ds_matrix_fh.puts %w[a b value].join("\t")
+    # Find all values in the database
+    seq2 = []
+    foreach_in_db(db, metric) do |r|
+      seq2 << r[0]
+      ds_matrix_fh.puts r[0,3].join("\t")
+    end
+    # Find all values among visited datasets in ref_project
+    ref_r = ref_project.result("#{metric}_distances") or return
+    Zlib::GzipReader.open(ref_r.file_path(:matrix)) do |fh|
+      fh.each_line do |ln|
+        r = ln.chomp.split("\t")
+        next unless seq2.include?(r[1]) or seq2.include?(r[2])
+        ds_matrix_fh.puts r[1,3].join("\t")
+      end
+    end
+    ds_matrix_fh.close
+    ref_tree = File.expand_path("utils/ref-tree.R", MiGA::MiGA.root_path)
+    `"#{ref_tree}" "#{ds_matrix}" "#{out_base}" "#{dataset.name}"`
+    File.unlink ds_matrix
+  end
+
+  # Tests taxonomy
+  def tax_test
+    # Get taxonomy of closest relative
+    from_ref_project = (project != ref_project)
+    res_dir = from_ref_project ?
+          File.expand_path("data/09.distances/05.taxonomy", project.path) : home
+    Dir.mkdir res_dir unless Dir.exist? res_dir
+    File.open(File.expand_path("#{dataset.name}.done", res_dir), "w") do |fh|
+      fh.puts Time.now.to_s
+    end
+    dataset.add_result(from_ref_project ? :taxonomy : :distances, true)
+    cr = dataset.closest_relatives(1, from_ref_project)
+    return if cr.nil? or cr.empty?
+    tax = ref_project.dataset(cr[0][0]).metadata[:tax] || {}
+    # Run the test for each rank
+    r = MiGA::TaxDist.aai_pvalues(cr[0][1], :intax).map do |k,v|
+      sig = ""
+      [0.5,0.1,0.05,0.01].each{ |i| sig << "*" if v<i }
+      [MiGA::Taxonomy.LONG_RANKS[k], (tax[k] || "?"), v, sig]
+    end
+    # Save test
+    File.open(File.expand_path("#{dataset.name}.intax.txt", home), "w") do |fh|
+      fh.puts MiGA::MiGA.tabulate(%w[Rank Taxonomy P-value Signif.], r)
+      fh.puts ""
+      fh.puts "Significance at p-value below: *0.5, **0.1, ***0.05, ****0.01."
+    end
+    return r
+  end
+
+  # Transfer the taxonomy to the current dataset
+  def transfer_taxonomy(tax)
+    pval = (project.metadata[:tax_pvalue] || 0.05).to_f
+    tax_a = tax.select{ |i| i[1]!="?" && i[2]<=pval }.map { |i| i[0,2].join(":") }
+    dataset.metadata[:tax] = MiGA::Taxonomy.new(tax_a)
+    dataset.save
+  end
+end

data/utils/distance/runner.rb

@@ -0,0 +1,104 @@
+
+require_relative 'base.rb'
+require_relative 'temporal.rb'
+require_relative 'database.rb'
+require_relative 'commands.rb'
+require_relative 'pipeline.rb'
+
+
+class MiGA::DistanceRunner
+
+  include MiGA::DistanceRunner::Temporal
+  include MiGA::DistanceRunner::Database
+  include MiGA::DistanceRunner::Commands
+  include MiGA::DistanceRunner::Pipeline
+
+  attr_reader :project, :ref_project, :dataset, :opts, :home
+  attr_reader :tmp, :tmp_dbs, :dbs, :db_counts
+
+  def initialize(project_path, dataset_name, opts_hash={})
+    @opts = opts_hash
+    @project = MiGA::Project.load(project_path) or
+          raise "No project at #{project_path}"
+    @dataset = project.dataset(dataset_name)
+    @home = File.expand_path("data/09.distances", project.path)
+    # Default opts
+    @opts[:aai_save_rbm] ||= ENV.fetch("MIGA_AAI_SAVE_RBM") do
+      project.is_clade? ? "save-rbm" : "no-save-rbm"
+    end
+    @opts[:thr] ||= ENV.fetch("CORES"){ 2 }.to_i
+    if opts[:run_taxonomy] && project.metadata[:ref_project]
+      @ref_project = MiGA::Project.load(project.metadata[:ref_project])
+    end
+    @ref_project ||= project
+    [:haai_p, :aai_p, :ani_p, :distances_checkpoint].each do |m|
+      @opts[m] ||= ref_project.metadata[m]
+    end
+    @opts[:distances_checkpoint] ||= 10
+    @opts[:distances_checkpoint] = @opts[:distances_checkpoint].to_i
+  end
+
+  # Launch the appropriate analysis
+  def go!
+    return if dataset.is_multi?
+    Dir.mktmpdir do |tmp_dir|
+      @tmp = tmp_dir
+      create_temporals
+      opts[:run_taxonomy] ? go_taxonomy! : dataset.is_ref? ? go_ref! : go_query!
+    end
+  end
+
+  # Launch analysis for reference datasets
+  def go_ref!
+    # Initialize databases
+    initialize_dbs! true
+    # first-come-first-serve traverse
+    ref_project.each_dataset do |ds|
+      next if !ds.is_ref? or ds.is_multi? or ds.result(:essential_genes).nil?
+      puts "[ #{Time.now} ] #{ds.name}"
+      aai = aai(ds)
+      ani(ds) unless aai.nil? or aai < 90.0
+    end
+    # Finalize
+    [:haai, :aai, :ani].each{ |m| checkpoint! m if db_counts[m] > 0 }
+  end
+
+  # Launch analysis for query datasets
+  def go_query!
+    # Check if project is ready
+    v = ref_project.is_clade? ? [:subclades, :ani] : [:clade_finding, :aai]
+    res = ref_project.result(v[0])
+    return if res.nil?
+    # Initialize the databases
+    initialize_dbs! false
+    # Calculate the classification-informed AAI/ANI traverse
+    results = File.expand_path("#{dataset.name}.#{v[1]}-medoids.tsv", home)
+    fh = File.open(results, "w")
+    classif, val_cls = *classify(res.dir, ".", v[1], fh)
+    fh.close
+    # Calculate all the AAIs/ANIs against the lowest subclade (if classified)
+    par_dir = File.dirname(File.expand_path(classif, res.dir))
+    par = File.expand_path("miga-project.classif", par_dir)
+    if File.size? par
+      File.open(par, "r") do |fh|
+        fh.each_line do |ln|
+          r = ln.chomp.split("\t")
+          next unless r[1].to_i==val_cls
+          target = ref_project.dataset(r[0])
+          aai = (metric==:aai) ? aai(target) : 100.0
+          ani(target) if aai >= 90.0
+        end
+      end
+    end
+    # Finalize
+    [:haai, :aai, :ani].each{ |m| checkpoint! m if db_counts[m] > 0 }
+    build_medoids_tree(v[1])
+    transfer_taxonomy(tax_test)
+  end
+
+  # Launch analysis for taxonomy jobs
+  def go_taxonomy!
+    return unless project.metadata[:ref_project]
+    go_query! # <- yeah, it's actually the same, just different ref_project
+  end
+end

data/utils/distance/temporal.rb

@@ -0,0 +1,37 @@
+
+require 'tmpdir'
+
+module MiGA::DistanceRunner::Temporal
+
+  # Copy input files to the (local) temporal folder
+  def create_temporals
+    rf = {essential_genes: :ess_genes, cds: :proteins, assembly: :largecontigs}
+    rf.each do |res, file|
+      r = dataset.result(res)
+      f = r.nil? ? nil : r.file_path(file)
+      FileUtils.cp(f, tmp_file("#{file}.fa")) unless f.nil?
+    end
+  end
+
+  # Temporal file with extension +ext+
+  def tmp_file(ext)
+    File.expand_path("#{dataset.name}.#{ext}", tmp)
+  end
+
+  # Copies temporal databases back to the MiGA Project if 10 or more values
+  # have been stored without copying. The period (10 by default) can be
+  # controlled using +@opts[:distances_checkpoint]+
+  def checkpoint(metric)
+    @db_counts[metric] += 1
+    checkpoint! metric if db_counts[metric] >= @opts[:distances_checkpoint]
+  end
+
+  # Copies temporal databases back to the MiGA Project
+  def checkpoint!(metric)
+    SQLite3::Database.new(tmp_dbs[metric]) do |conn|
+      conn.execute("select count(*) from #{metric==:haai ? :aai : metric}")
+    end
+    FileUtils.cp(tmp_dbs[metric], dbs[metric])
+    @db_counts[metric] = 0
+  end
+end

data/utils/distances.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require_relative 'distance/runner.rb'
+
+dataset = ARGV.shift
+project = ARGV.shift
+opts = Hash[ ARGV.map{ |i| i.split("=",2).tap{ |j| j[0] = j[0].to_sym } } ]
+runner = MiGA::DistanceRunner.new(dataset, project, opts)
+runner.go!

data/utils/enveomics/Docs/recplot2.md

@@ -0,0 +1,233 @@
+# Recruitment plots
+
+## Aims
+
+This document aims to cover the technical aspects of the recruitment plot functions in the
+`enveomics.R` package, focusing on the peak finder and gene-content diversity analyses.
+
+## Caveats
+
+This is a __*working document*__, describing  unstable and/or experimental code. The material
+here is susceptible of changes without warning, pay attention to the modification date and (if
+in doubt) the commit history. The definitions and default parameters of the functions described
+here may change in the near future as result of further experimentation or more stable
+implementations.
+
+The current document was generated and tested with the `enveomics.R` package version 1.3. To
+check your current version in R, use `packageVersion('enveomics.R')`.
+
+> **IMPORTANT**: Some of the functions described here may return unexpected results with your data.
+> Carefully evaluate all your results.
+
+---
+
+## Package: `enveomics.R`
+
+The functionalities described here are provided by the `enveomics.R` package. Some features
+described here are updated more frequently than the official
+[CRAN releases](https://CRAN.R-project.org/package=enveomics.R). In order to have the latest
+updates (package HEAD), download (or update), and install this git repository.
+
+### Quick installation guide
+
+:globe_with_meridians: To install the latest stable version available in CRAN, use in R:
+
+```R
+install.packages(c('enveomics.R','optparse'))
+```
+
+:octocat: To install the latest HEAD version (potentially unstable) available in GitHub, use in R:
+
+```R
+install.packages('devtools')
+library('devtools')
+install_github('lmrodriguezr/enveomics', subdir='enveomics.R')
+```
+
+---
+
+## Recruitment plots: `enve.recplot2`
+
+The first step in this analysis is the mapping of reads to the genome, processed with
+[BlastTab.catsbj.pl](http://enve-omics.ce.gatech.edu/enveomics/docs?t=BlastTab.catsbj.pl).
+We'll assume the mapping is saved in the file `my-mapping.tab` and this is also the
+prefix of the processed files.
+
+Once you have these input files (`.rec` and `.lim`), you can build the recruitment plot.
+For this, you'll have two options.
+
+### Option 1: Using the `BlastTab.recplot2.R` stand-alone script
+
+The stand-alone script
+[BlastTab.recplot2.R](http://enve-omics.ce.gatech.edu/enveomics/docs?t=BlastTab.recplot2.R)
+is the easiest option to run, and should be the preferred method if you're automating
+this analysis to process several mappings, but it doesn't offer access to advanced options.
+
+You can run it like this using two CPUs:
+
+```bash
+BlastTab.recplot2.R --prefix my-mapping.tab --threads 2 my-recplot.rdata my-recplot.pdf
+```
+
+> **NOTE 1**: It's NOT recommended to map reads against genes, the recommended strategy is to
+> map against contigs. However, if you did map reads against genes, you may want to use the
+> `--pos-breaks 0` option to use each gene as a recruitment window.
+> 
+> **NOTE 2**: If you want to plot the population peaks at this step, simply pass the
+> `--peaks-col darkred` option.
+
+Now you should have two output files: `my-recplot.rdata`, containing your `enve.RecPlot2` R
+object, and `my-recplot.pdf` with the graphical output of the recruitment plot.
+
+### Option 2: Using the `enve.recplot2` R function
+
+If you require access to advanced options, or for some other reason prefer to calculate the
+recruitment plot interactively, you can directly use the `enve.recplot2` R function. This is
+and example session in R:
+
+```R
+# Load the package
+library(enveomics.R)
+# Open the PDF
+pdf('my-recplot.pdf')
+# Build and plot the object using two threads and no peak detection
+# (to turn on peak detection, simply remove `peaks.col=NA`)
+rp <- enve.recplot2('my-mapping.tab', threads=2, peaks.col=NA)
+# Close the PDF
+dev.off()
+# Save the object
+save(rp, file='my-recplot.rdata')
+```
+
+> **IMPORTANT**: Remember to save the `enve.RecPlot2` R object (that's the last line above)
+> before closing the R session.
+
+Naturally, you may want to see what other (advanced) options you have. You can access the
+documentation of the function in R using `?enve.recplot2`.
+
+---
+
+## Summary statistics
+
+Here we explore some frequently used summary statistics from recruitment plots. First, load the
+package and the `enve.RecPlot2` object you saved previously, in R:
+
+```R
+library(enveomics.R)
+load('my-recplot.rdata')
+```
+
+### Average and median sequencing depth
+
+```R
+mean(enve.recplot2.seqdepth(rp)) # <- Average
+median(enve.recplot2.seqdepth(rp)) # <- Median
+```
+
+### Average and median sequencing depth excluding zero-coverage windows
+
+```R
+seqdepth <- enve.recplot2.seqdepth(rp)
+mean(seqdepth[seqdepth>0]) # <- Average
+median(seqdepth[seqdepth>0]) # <- Median
+```
+
+### Average Nucleotide Identity from reads (ANIr)
+
+```R
+enve.recplot2.ANIr(rp) # <- Complete recruitment plot
+enve.recplot2.ANIr(rp, c(90,100)) # <- All reads above 90% (recommended for intra-population)
+enve.recplot2.ANIr(rp, c(95,100)) # <- Reads above 95%
+enve.recplot2.ANIr(rp, c( 0, 90)) # <- Between populations (other species)
+```
+
+### Coordinates of each sequence window with their respective sequencing depth
+
+```R
+d <- enve.recplot2.coordinates(rp)
+d$seqdepth <- enve.recplot2.seqdepth(rp)
+d
+```
+
+### Sequencing breadth (upper boundary)
+
+This estimate depends on the window size. The smaller the window size, the better the
+estimate. When the window size is 1bp, the estimate is exact, otherwise it's consistently
+biased (overestimate).
+
+```R
+mean(enve.recplot2.seqdepth(rp) > 0)
+```
+
+---
+
+## Peak-finder: `enve.recplot2.findPeaks`
+
+In this step we will try to identify one or multiple population peaks corresponding to different
+sub-populations and/or composites of sub-populations.
+
+> **NOTE** This step can be performed together with the step above, but we separate it here for
+> two reasons: **(1)** This step is much more unstable but less computationally demanding than the
+> step before, so it makes sense to re-run only this part with different parameters and/or
+> package updates; and **(2)** We want to save the R objects independently, so the following steps
+> are more clear.
+
+In R:
+
+```R
+# Load the package
+library(enveomics.R)
+# Load the `enve.RecPlot2` object you saved previously
+load('my-recplot.rdata')
+# Find the peaks
+peaks <- enve.recplot2.findPeaks(rp)
+# Save the peaks R object (optional)
+save(peaks, file='my-recplot-peaks.rdata')
+# Plot the peaks in a PDF (optional)
+pdf('my-recplot-peaks.pdf')
+p <- plot(rp, use.peaks=peaks, layout=4) # <- Remove `layout=4` for the full plot
+dev.off()
+```
+
+The key function here is `enve.recplo2.findPeaks`. This function has several parameters, depending on
+the method used. To see all supported methods, use `?enve.recplot2.findPeaks`. To see all the options
+of the default method (`'emauto'`) use `?enve.recplot2.findPeaks.emauto`.
+
+---
+
+## Gene-content diversity: `enve.recplot2.extractWindows`
+
+In R:
+
+```R
+# Load the package and the objects (unless you're still in the same session from the last step)
+library(enveomics.R)
+load('my-recplot.rdata')
+load('my-recplot-peaks.rdata')
+# Find the peak representing the core genome
+cp <- enve.recplot2.corePeak(peaks)
+#-----
+# The following functions illustrate how to obtain different results. Please explore the resulting
+# objects and the associated documentation
+#-----
+# Find the coordinates of windows significantly below the average sequencing depth
+div <- enve.recplot2.extractWindows(rp, cp, seq.names=TRUE)
+# Add sequencing depth
+div$seqdepth <- enve.recplot2.seqdepth(rp, as.numeric(rownames(div)))
+# Save the coordinates as a tab-delimited table
+write.table(div, 'my-low-seqdepth.tsv', quote=FALSE, sep='\t', row.names=FALSE)
+# Find all the windows with sequencing depth zero
+zero <- enve.recplot2.coordinates(rp, enve.recplot2.seqdepth(rp)==0)
+```
+
+---
+
+## To do
+
+- [x] Document structure
+- [x] Package: `enveomics.R`
+- [x] Recruitment plots: `enve.recplot2`
+- [x] Summary statistics
+- [x] Peak-finder: `enve.recplot2.findPeaks`
+- [x] Gene-content diversity: `enve.recplot2.extractWindows`
+- [ ] Compare identity profiles: `enve.recplot2.compareIdentities`