miga-base 0.2.1.0 → 0.2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f914d4f05c7c1cb97b6036fa1af3bb5c6e95c341
-  data.tar.gz: 3812c256d78cbe8125ee5aa14a1785c3de57918f
+  metadata.gz: 5a495f0a540632069724a9f30b87495532c74aa5
+  data.tar.gz: 21e81ff7c152fbc6ef89e781b25aef2852190622
 SHA512:
-  metadata.gz: cdc6692724bb6b7c39a313c35eaabe51405d104d8af465618983a83601435af66ca1f7eca0a3e9fbc19b24ce13cbde8f9962beaf643ad1f9e3970bb10067f671
-  data.tar.gz: 45e19af2b3e14e225c9f6a89193b0c400fe8ac685296314b4320885a563ab2b3c4472563a45f98c5a7d15edd74573cf1f9176a59905a2764a809d6bc3850b90d
+  metadata.gz: 7f0f38d7981e0f872af676b4e65341baa032a00b7dd491337b1c01bf72c1c4a9702b5808ce2eb19486d1c139658e303c1d39b9422f0ae6301e4af9dc4be5ce33
+  data.tar.gz: 8b731945f0dc75f9f29c6dc92f943ad069e42038c6aec6fe64885832ce3ceeb1dad824b5d594ca03d25febb5d5f70599f7049ce2739c2892e8c2a0d330c955f9

data/actions/tax_distributions.rb

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+require "miga/tax_index"
+require "zlib"
+require "tmpdir"
+
+o = {q:true, format: :json}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project])
+  opt_filter_datasets(opt, o)
+  opt.on("-i", "--index FILE",
+    "Pre-calculated tax-index (in tabular format) to be used.",
+    "If passed, dataset filtering arguments are ignored."
+    ){ |v| o[:index]=v }
+  opt_common(opt, o)
+end.parse!
+
+##=> Functions <=
+# Returns the _cannonical_ ID between strings +a+ and +b+.
+def cannid(a, b) ; [a, b].sort.join("-") ; end
+
+##=> Main <=
+opt_require(o, project:"-P")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+metric = p.is_clade? ? "ani" : "aai"
+res_n  = "#{metric}_distances"
+$stderr.puts "Reading distances (1-#{metric.upcase})." unless o[:q]
+res = p.result res_n
+raise "#{res_n} not yet calculated." if res.nil?
+matrix = res.file_path(:matrix)
+raise "#{res_n} has no matrix." if matrix.nil?
+dist = {}
+fh = matrix=~/\.gz$/ ? Zlib::GzipReader.open(matrix) : File.open(matrix,"r")
+fh.each_line do |ln|
+  next if fh.lineno==1
+  row = ln.chomp.split(/\t/)
+  dist[cannid(row[1], row[2])] = [row[3], 0, ["root:biota"]]
+end
+fh.close
+
+Dir.mktmpdir do |dir|
+  if o[:index].nil?
+    $stderr.puts "Loading datasets." unless o[:q]
+    ds = p.datasets
+    ds.select!{ |d| not d.metadata[:tax].nil? }
+    ds = filter_datasets!(ds, o)
+    
+    $stderr.puts "Indexing taxonomy." unless o[:q]
+    tax_index = MiGA::TaxIndex.new
+    ds.each { |d| tax_index << d }
+    tab = File.expand_path("index.tab", dir)
+    File.open(tab, "w") { |fh| fh.print tax_index.to_tab }
+  else
+    tab = o[:index]
+  end
+  
+  $stderr.puts "Traversing taxonomy." unless o[:q]
+  rank_i = 0
+  MiGA::Taxonomy.KNOWN_RANKS.each do |rank|
+    $stderr.print "o #{rank}: " unless o[:q]
+    rank_n = 0
+    rank_i += 1
+    in_rank = nil
+    ds_name = []
+    File.open(tab, "r") do |fh|
+      fh.each_line do |ln|
+        if ln =~ /^ {#{(rank_i-1)*2}}\S+:\S+:/
+          in_rank = nil
+          ds_name = []
+        elsif ln =~ /^ {#{rank_i*2}}(#{rank}:(\S+)):/
+          in_rank = $2=="?" ? nil : $1
+          ds_name = []
+        elsif ln =~ /^ *# (\S+)/ and not in_rank.nil?
+          ds_i = $1
+          ds_name << ds_i
+          ds_name.each do |ds_j|
+            k = cannid(ds_i, ds_j)
+            next if dist[k].nil?
+            rank_n += 1
+            dist[k][1] = rank_i
+            dist[k][2].unshift in_rank
+          end
+        end
+      end
+    end
+    $stderr.puts "#{rank_n} pairs of datasets." unless o[:q]
+  end
+end
+
+dist.keys.each do |k|
+  puts (k.split("-") + dist[k].flatten).join("\t")
+end

data/bin/miga

@@ -26,6 +26,7 @@ $task_desc = {
   list_files: "Lists all registered files from the results of a dataset or a "+
     "project.",
   project_info: "Displays information about a MiGA project.",
+  tax_distributions: "Estimates distributions of distance by taxonomy.",
   unlink_dataset: "Removes a dataset from an MiGA project."
 }
 

data/lib/miga/tax_dist.rb

@@ -0,0 +1,65 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require "miga/common"
+require "miga/taxonomy"
+require "zlib"
+
+##
+# Methods for taxonomy identification based on AAI/ANI values.
+module MiGA::TaxDist
+  
+  ##
+  # Absolute path to the :intax or :novel data file (determined by +test+) for
+  # AAI.
+  def self.aai_path(test)
+    test = test.downcase.to_sym
+    return nil unless [:intax, :novel].include? test
+    File.expand_path("../_data/aai-#{test}.tsv.gz", __FILE__)
+  end
+
+  # Returns a Hash, where the keys correspond to the taxonomic level
+  # (see MiGA::Taxonomy.LONG_RANKS for the meanings), and the values correspond
+  # to the p-values of being :intax or :novel, as determined by +test+.
+  def self.aai_pvalues(aai, test)
+    Zlib::GzipReader.open(aai_path(test)) do |fh|
+      keys = nil
+      fh.each_line do |ln|
+        row = ln.chomp.split /\t/
+        if fh.lineno==1
+          keys = row[1, row.size-1].map{ |i| i.to_i }
+        elsif row.shift.to_f >= aai
+          vals = {}
+          keys.each do |i|
+            v = row.shift
+            next if v=="NA"
+            vals[MiGA::Taxonomy.KNOWN_RANKS[i]] = v.to_f
+          end
+          return vals
+        end
+      end # each_line ln
+    end # open fh
+    {}
+  end
+
+  # Determines the degree to which a Float +aai+ value indicates similar
+  # taxonomy (with +test+ :intax) or a novel taxon (with +test+ :novel). Returns
+  # a Hash with "likelihood" phrases as keys and values as an array with
+  # cannonical rank (as in MiGA::Taxonomy) and estimated p-value.
+  def self.aai_taxtest(aai, test)
+    meaning = {most_likely:[0,0.01],probably:[0.01,0.1],possibly_even:[0.1,0.5]}
+    pv = aai_pvalues(aai, test)
+    out = {}
+    meaning.each do |phrase, thresholds|
+      lwr, upr = thresholds
+      min = pv.values.select{ |v| v >= lwr }.min
+      return out if min.nil?
+      if min < upr
+        v = pv.select{ |_,v| v==min }
+        out[phrase] = (test==:intax ? v.reverse_each : v).first
+      end
+    end
+    out
+  end
+
+end

data/lib/miga/tax_index.rb

@@ -34,6 +34,26 @@ class MiGA::TaxIndex < MiGA::MiGA
     @datasets << dataset
   end
 
+  ##
+  # Finds all the taxa in the collection at the +rank+ taxonomic rank.
+  def taxa_by_rank(rank)
+    rank = MiGA::Taxonomy.normalize_rank(rank)
+    taxa = [@root]
+    select = []
+    loop do
+      new_taxa = []
+      taxa.map{ |tx| tx.children }.flatten.each do |ch|
+        if ch.rank == rank
+          select << ch
+        elsif not ch.children.empty?
+          new_taxa << ch
+        end
+      end
+      break if new_taxa.empty?
+    end
+    select
+  end
+
   ##
   # Generate JSON String for the index.
   def to_json
@@ -98,6 +118,12 @@ class MiGA::TaxIndexTaxon < MiGA::MiGA
     datasets.size + children.map{ |it| it.datasets_count }.reduce(0, :+)
   end
 
+  ##
+  # Get all the datasets in the taxon (including children).
+  def all_datasets
+    datasets + children.map{ |it| it.datasets }.reduce([], :+)
+  end
+
   ##
   # JSON String of the taxon.
   def to_json(*a)

data/lib/miga/taxonomy.rb

@@ -11,6 +11,13 @@ class MiGA::Taxonomy < MiGA::MiGA
   def self.KNOWN_RANKS() @@KNOWN_RANKS ; end
   @@KNOWN_RANKS = %w{ns d k p c o f g s ssp str ds}.map{|r| r.to_sym}
 
+  ##
+  # Long names of the cannonical ranks.
+  def self.LONG_RANKS() @@LONG_RANKS ; end
+  @@LONG_RANKS = {root: "root", ns: "namespace", d: "domain", k: "kingdom",
+    p: "phylum", c: "class", o: "order", f: "family", g: "genus", s: "species",
+    ssp: "subspecies", str: "strain", ds: "dataset"}
+
   ##
   # Synonms for cannonical ranks.
   @@RANK_SYNONYMS = {

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.2, 1, 0]
+  VERSION = [0.2, 1, 2]
   
   ##
   # Nickname for the current major.minor version.

data/utils/plot-taxdist.R

@@ -0,0 +1,110 @@
+#!/usr/bin/env Rscript
+
+# @package MiGA
+# @license Artistic-2.0
+
+##
+# To update the AAI data files, use:
+# 
+# ```bash
+# miga tax_distributions -P /Path/To/RefSeq --ref | cut -f 1-5 \
+#   > aai-tax-index.tsv
+# ```
+# 
+# Next, in R:
+# 
+# ```R
+# source("utils/plot-taxdist.R")
+# p.val <- plot.miga.taxdist("aai-tax-index.tsv",
+#   exclude=c("g:Mycoplasma", "ssp:Prochlorococcus_marinus_subsp__marinus",
+#   "f:Rhizobiaceae", "s:Buchnera_aphidicola", "s:Prochlorococcus_marinus"))
+# write.table(p.val[[1]], file="lib/miga/_data/aai-intax.tsv",
+#   sep="\t", row.names=TRUE, col.names=NA, quote=FALSE)
+# write.table(p.val[[2]], file="lib/miga/_data/aai-novel.tsv",
+#   sep="\t", row.names=TRUE, col.names=NA, quote=FALSE)
+# ```
+#
+# And finally, back in bash:
+# 
+# ```bash
+# gzip lib/miga/_data/aai-intax.tsv
+# gzip lib/miga/_data/aai-novel.tsv
+# rm aai-tax-index.tsv
+# ```
+#
+
+#= Load stuff
+#argv <- commandArgs(trailingOnly=T)
+
+#= Functions
+plot.miga.taxdist <- function(file, exclude=c()){
+  pdf(paste(file, ".pdf", sep=""), 6, 7)
+  layout(1:3, heights=c(2,1,1.5))
+  d <- read.table(file, sep="\t", header=FALSE,
+    col.names=c("a","b","aai","rank","taxon"), as.is=TRUE)
+  a <- d[!(d$taxon %in% exclude),]
+  col <- rainbow(max(a$rank)+1, s=3/4, v=3/4, alpha=1/3)
+  col2 <- rainbow(max(a$rank)+1, s=3/4, v=3/4)
+
+  cat("o Plot pairs.\n")
+  par(mar=c(0,4,1,1)+0.1)
+  plot(d$aai, d$rank+runif(nrow(d), -0.3, 0.3), cex=1/2, pch=16, las=1, bty="l",
+    col=ifelse(d$taxon %in% exclude, rgb(.8,.8,.8,1/4), col[d$rank+1]),
+    xlab="", ylab="Lowest common taxon", xaxt="n", ylim=rev(range(a$rank)))
+  for(i in c(0.1, 0.05, 0.01)){
+    min_q <- tapply(a$aai, a$rank, quantile, probs=i)
+    max_q <- tapply(a$aai, a$rank, quantile, probs=1-i)
+    arrows(x0=min_q, length=0, col=grey(1+log10(i)/2),
+      y0=as.numeric(names(min_q))-0.45, y1=as.numeric(names(min_q))+0.45)
+    arrows(x0=max_q, length=0, col=grey(1+log10(i)/2),
+      y0=as.numeric(names(max_q))-0.45, y1=as.numeric(names(max_q))+0.45)
+  }
+  
+  cat("o Plot taxa.\n")
+  par(mar=c(0,4,0,1)+0.1)
+  plot(1, type="n", xlim=range(a$aai), ylim=rev(range(a$rank)),
+    xlab="", ylab="Lowest common taxon", xaxt="n", las=1, bty="l")
+  for(i in unique(a$rank)){
+    t.aai <- tapply(d$aai[d$rank==i], d$taxon[d$rank==i], mean)
+    t.size <- tapply(d$aai[d$rank==i], d$taxon[d$rank==i], length)
+    points(t.aai, i+runif(length(t.aai), -0.15, 0.15), pch=16,
+      cex=2*log2(1+t.size)/log2(1+max(t.size)),
+      col=ifelse(names(t.aai) %in% exclude, rgb(.8,.8,.8,1/4), col[i+1]))
+  }
+  
+  cat("o Plot p-values.\n")
+  par(mar=c(4,4,0,1)+0.1)
+  plot(1, type="n", xlim=range(a$aai), ylim=c(0,0.5),
+    xlab="AAI (%)", ylab="P-value (--- intax; - - novel)", las=1, bty="l")
+  x <- seq(30, 100, 0.1)
+  intax <- data.frame(row.names=x)
+  novel <- data.frame(row.names=x)
+  for(i in sort(unique(a$rank))){
+    if(i==12) next
+    k <- as.character(i)
+    intax[,k] <- miga.taxprob.intax(x, i, a)
+    novel[,k] <- miga.taxprob.novel(x, i, a)
+    lines(x, intax[,k], col=col2[i+1], lwd=2)
+    lines(x, novel[,k], col=col2[i+1], lwd=2, lty=2)
+  }
+  dev.off()
+  return(list(intax, novel))
+}
+
+miga.taxprob.novel <- function(max.aai, rank, data){
+  o <- c()
+  for(i in max.aai){
+    a <- sum(data$rank >= rank & data$aai <= i)/sum(data$aai <= i)
+    o <- c(o, a)
+  }
+  return(o*sum(data$rank < 12)/sum(data$rank >= rank))
+}
+
+miga.taxprob.intax <- function(max.aai, rank, data){
+  o <- c()
+  for(i in max.aai){
+    a <- sum(data$rank < rank & data$aai >= i)/sum(data$aai >= i)
+    o <- c(o, a)
+  }
+  return(o*sum(data$rank < 12)/sum(data$rank < rank))
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.2.1.0
+  version: 0.2.1.2
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
@@ -14,70 +14,70 @@ dependencies:
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.8'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '11'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3'
 description: Microbial Genomes Atlas
@@ -88,26 +88,6 @@ extensions: []
 extra_rdoc_files:
 - README.md
 files:
-- Gemfile
-- LICENSE
-- README.md
-- Rakefile
-- actions/add_result.rb
-- actions/add_taxonomy.rb
-- actions/create_dataset.rb
-- actions/create_project.rb
-- actions/daemon.rb
-- actions/date.rb
-- actions/download_dataset.rb
-- actions/find_datasets.rb
-- actions/import_datasets.rb
-- actions/index_taxonomy.rb
-- actions/list_datasets.rb
-- actions/list_files.rb
-- actions/project_info.rb
-- actions/unlink_dataset.rb
-- bin/miga
-- lib/miga.rb
 - lib/miga/common.rb
 - lib/miga/daemon.rb
 - lib/miga/dataset.rb
@@ -116,9 +96,22 @@ files:
 - lib/miga/project.rb
 - lib/miga/remote_dataset.rb
 - lib/miga/result.rb
+- lib/miga/tax_dist.rb
 - lib/miga/tax_index.rb
 - lib/miga/taxonomy.rb
 - lib/miga/version.rb
+- lib/miga.rb
+- test/common_test.rb
+- test/daemon_test.rb
+- test/dataset_test.rb
+- test/metadata_test.rb
+- test/project_test.rb
+- test/remote_dataset_test.rb
+- test/tax_index_test.rb
+- test/taxonomy_test.rb
+- test/test_helper.rb
+- lib/miga/_data/aai-intax.tsv.gz
+- lib/miga/_data/aai-novel.tsv.gz
 - scripts/_distances_functions.bash
 - scripts/_distances_noref_nomulti.bash
 - scripts/_distances_ref_nomulti.bash
@@ -140,21 +133,33 @@ files:
 - scripts/subclades.bash
 - scripts/trimmed_fasta.bash
 - scripts/trimmed_reads.bash
-- test/common_test.rb
-- test/daemon_test.rb
-- test/dataset_test.rb
-- test/metadata_test.rb
-- test/project_test.rb
-- test/remote_dataset_test.rb
-- test/tax_index_test.rb
-- test/taxonomy_test.rb
-- test/test_helper.rb
 - utils/adapters.fa
 - utils/mytaxa_scan.R
 - utils/mytaxa_scan.rb
+- utils/plot-taxdist.R
 - utils/requirements.txt
 - utils/subclades-compile.rb
 - utils/subclades.R
+- bin/miga
+- actions/add_result.rb
+- actions/add_taxonomy.rb
+- actions/create_dataset.rb
+- actions/create_project.rb
+- actions/daemon.rb
+- actions/date.rb
+- actions/download_dataset.rb
+- actions/find_datasets.rb
+- actions/import_datasets.rb
+- actions/index_taxonomy.rb
+- actions/list_datasets.rb
+- actions/list_files.rb
+- actions/project_info.rb
+- actions/tax_distributions.rb
+- actions/unlink_dataset.rb
+- Gemfile
+- Rakefile
+- README.md
+- LICENSE
 homepage: http://enve-omics.ce.gatech.edu/miga
 licenses:
 - Artistic-2.0
@@ -163,25 +168,25 @@ post_install_message:
 rdoc_options:
 - lib
 - README.md
-- "--main"
+- --main
 - README.md
-- "--title"
+- --title
 - MiGA
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: MiGA