miga-base 0.2.4.3 → 0.2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fdd460c0009e55ffc82547e7d31f94789a874442
-  data.tar.gz: c3e560fb2e8871ac586ef8b1fd64c2b1b2076471
+  metadata.gz: 367ffd0c85c1ad5bdbc44f4c88677fac5268c2e2
+  data.tar.gz: eb107b14279ea252e4287d40efa73a562d890451
 SHA512:
-  metadata.gz: c77bb9fc35f046a2ebc1c1ddc457f19c4accafe06958b43d42a3654f91a5c19dcf92cc3624f9b30b950542d736787dc69f2e8e9ba88141d20923823da0812f46
-  data.tar.gz: ecee7c4611a74fd16115260e706ea19315ccd639070482010f0bf5c4f669fe30fe9dea755b912053136366831f715cf346f2a32ed38dbee1ea69442525f9e080
+  metadata.gz: 7a2bf0f148315cd6cc0fd57e5ff48a6f88cb5be7ebcaeea4e3b6a0b820cc71fca1165f98287fcd94ffd5b330e3e352281c9d517d66b2df3219a4af067854c0e4
+  data.tar.gz: c8d884b0c56d075179c38470e4d0a34316ad14706c9a88ac171fc0230811e863090ce3a7ab0330c5c871d6754f15c01cae6b177cb6cf68b13153c3979934ff2a

data/actions/result_stats.rb

@@ -26,6 +26,7 @@ p = MiGA::Project.load(o[:project])
 raise "Impossible to load project: #{o[:project]}" if p.nil?
 
 $stderr.puts "Loading result." unless o[:q]
+d = nil
 if o[:dataset].nil?
   r = p.add_result(o[:name], false)
 else
@@ -77,6 +78,8 @@ if o[:compute]
       end
     end
     stats[:quality] = stats[:completeness][0] - stats[:contamination][0]*5
+  when :distances
+    d.cleanup_distances! unless d.nil?
   else
     stats = nil
   end

data/lib/miga/dataset.rb

@@ -198,6 +198,11 @@ class MiGA::Dataset < MiGA::MiGA
     r
   end
 
+  ##
+  # Gets a result as MiGA::Result for the datasets with +result_type+. This is
+  # equivalent to +add_result(result_type, false)+.
+  def get_result(result_type) ; add_result(result_type, false) ; end
+
   ##
   # Returns the key symbol of the first registered result (sorted by the
   # execution order). This typically corresponds to the result used as the

data/lib/miga/dataset_result.rb

@@ -1,8 +1,29 @@
 
+require "sqlite3"
+
 ##
 # Helper module including specific functions to add dataset results.
 module MiGA::DatasetResult
   
+  ##
+  # Clean-up all the stored distances, removing values for datasets no longer in
+  # the project as reference datasets.
+  def cleanup_distances!
+    r = get_result(:distances)
+    return if r.nil?
+    [:haai_db, :aai_db, :ani_db].each do |db_type|
+      db = r.file_path(db_type)
+      next if db.nil? or not File.size? db
+      sqlite_db = SQLite3::Database.new db
+      table = db_type[-6..-4]
+      val = sqlite_db.execute "select seq2 from #{table}"
+      next if val.empty?
+      (val.map{ |i| i.first } - project.dataset_names).each do |extra|
+        sqlite_db.execute "delete from #{table} where seq2=?", extra
+      end
+    end
+  end
+
   private
 
     ##

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.2, 4, 3]
+  VERSION = [0.2, 5, 0]
   
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
   
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2017, 3, 28)
+  VERSION_DATE = Date.new(2017, 3, 29)
   
   ##
   # Reference of MiGA.

data/scripts/stats.bash

@@ -13,7 +13,7 @@ cd "$DIR"
 date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.start"
 
 # Calculate statistics
-for i in raw_reads trimmed_fasta assembly cds essential_genes ; do
+for i in raw_reads trimmed_fasta assembly cds essential_genes distances ; do
   echo "# $i"
   miga result_stats --compute-and-save -P "$PROJECT" -D "$DATASET" -r $i
 done

metadata

@@ -1,83 +1,103 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.2.4.3
+  version: 0.2.5.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-28 00:00:00.000000000 Z
+date: 2017-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.4
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '11'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
 description: Microbial Genomes Atlas
@@ -88,6 +108,31 @@ extensions: []
 extra_rdoc_files:
 - README.md
 files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- actions/add_result.rb
+- actions/add_taxonomy.rb
+- actions/create_dataset.rb
+- actions/create_project.rb
+- actions/daemon.rb
+- actions/date.rb
+- actions/download_dataset.rb
+- actions/find_datasets.rb
+- actions/import_datasets.rb
+- actions/index_taxonomy.rb
+- actions/list_datasets.rb
+- actions/list_files.rb
+- actions/plugins.rb
+- actions/project_info.rb
+- actions/result_stats.rb
+- actions/tax_distributions.rb
+- actions/unlink_dataset.rb
+- bin/miga
+- lib/miga.rb
+- lib/miga/_data/aai-intax.tsv.gz
+- lib/miga/_data/aai-novel.tsv.gz
 - lib/miga/common.rb
 - lib/miga/daemon.rb
 - lib/miga/dataset.rb
@@ -101,18 +146,6 @@ files:
 - lib/miga/tax_index.rb
 - lib/miga/taxonomy.rb
 - lib/miga/version.rb
-- lib/miga.rb
-- test/common_test.rb
-- test/daemon_test.rb
-- test/dataset_test.rb
-- test/metadata_test.rb
-- test/project_test.rb
-- test/remote_dataset_test.rb
-- test/tax_index_test.rb
-- test/taxonomy_test.rb
-- test/test_helper.rb
-- lib/miga/_data/aai-intax.tsv.gz
-- lib/miga/_data/aai-novel.tsv.gz
 - scripts/_distances_functions.bash
 - scripts/_distances_noref_nomulti.bash
 - scripts/_distances_ref_nomulti.bash
@@ -135,6 +168,15 @@ files:
 - scripts/subclades.bash
 - scripts/trimmed_fasta.bash
 - scripts/trimmed_reads.bash
+- test/common_test.rb
+- test/daemon_test.rb
+- test/dataset_test.rb
+- test/metadata_test.rb
+- test/project_test.rb
+- test/remote_dataset_test.rb
+- test/tax_index_test.rb
+- test/taxonomy_test.rb
+- test/test_helper.rb
 - utils/adapters.fa
 - utils/mytaxa_scan.R
 - utils/mytaxa_scan.rb
@@ -142,30 +184,7 @@ files:
 - utils/requirements.txt
 - utils/subclades-compile.rb
 - utils/subclades-nj.R
-- utils/subclades-pam.R
 - utils/subclades.R
-- bin/miga
-- actions/add_result.rb
-- actions/add_taxonomy.rb
-- actions/create_dataset.rb
-- actions/create_project.rb
-- actions/daemon.rb
-- actions/date.rb
-- actions/download_dataset.rb
-- actions/find_datasets.rb
-- actions/import_datasets.rb
-- actions/index_taxonomy.rb
-- actions/list_datasets.rb
-- actions/list_files.rb
-- actions/plugins.rb
-- actions/project_info.rb
-- actions/result_stats.rb
-- actions/tax_distributions.rb
-- actions/unlink_dataset.rb
-- Gemfile
-- Rakefile
-- README.md
-- LICENSE
 homepage: http://enve-omics.ce.gatech.edu/miga
 licenses:
 - Artistic-2.0
@@ -174,25 +193,25 @@ post_install_message:
 rdoc_options:
 - lib
 - README.md
-- --main
+- "--main"
 - README.md
-- --title
+- "--title"
 - MiGA
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: MiGA

data/utils/subclades-pam.R

@@ -1,186 +0,0 @@
-#!/usr/bin/env Rscript
-#
-# @package MiGA
-# @license Artistic-2.0
-#
-
-#= Load stuff
-argv <- commandArgs(trailingOnly=T)
-suppressPackageStartupMessages(library(ape))
-suppressPackageStartupMessages(library(vegan))
-suppressPackageStartupMessages(library(cluster))
-suppressPackageStartupMessages(library(parallel))
-suppressPackageStartupMessages(library(enveomics.R))
-
-#= Main function
-subclades <- function(ani_file, out_base, thr=1, ani=c()) {
-  say("==> Out base:", out_base, "<==")
-  
-  # Input arguments
-  if(missing(ani_file)){
-    a <- as.data.frame(ani)
-  }else{
-    a <- read.table(gzfile(ani_file), sep="\t", header=TRUE, as.is=TRUE)
-  }
-  if(nrow(a)==0){
-    generate_empty_files(out_base)
-    return(NULL)
-  }
-  
-  # Get ANI distances
-  say("Distances")
-  a$d <- 1-a$value/100
-  ani.d <- enve.df2dist(data.frame(a$a, a$b, a$d), default.d=max(a$d)*1.2)
-  ani.ph <- bionj(ani.d)
-  express.ori <- options('expressions')$expressions
-  if(express.ori < ani.ph$Nnode*4){
-    options(expressions=min(c(5e7,ani.ph$Nnode*4)))
-  }
-  write.tree(ani.ph, paste(out_base, ".nwk", sep=""))
-  options(expressions=express.ori)
-  
-  # Silhouette
-  say("Silhouette")
-  k <- 2:min(length(labels(ani.d))-1, 100)
-  cl <- makeCluster(thr)
-  s <- parSapply(cl, k, function(x) {
-      library(cluster)
-      s <- pam(ani.d, x, do.swap=FALSE, pamonce=1)$silinfo
-      c(s$avg.width, -sum(ifelse(s$widths[,3]>0,0,s$widths[,3])))
-    })
-  stopCluster(cl)
-  s.avg.z <- (s[1,]-mean(s[1,]))/(sd(s[1,])+0.0001)
-  s.neg.z <- (s[2,]-mean(s[2,]))/(sd(s[2,])+0.01)
-  ds <- s.avg.z - s.neg.z - 2/(1:length(k)) - (1:length(k))/50
-  top.n <- k[which.max(ds)]
-  
-  # Classify genomes
-  say("Classify => k :", top.n, "| n :", length(labels(ani.d)))
-  ani.cl <- pam(ani.d, top.n, pamonce=1)
-  ani.types <- ani.cl$clustering
-  ani.medoids <- ani.cl$medoids
-  
-  # Generate graphic report
-  say("Graphic report")
-  pdf(paste(out_base, ".pdf", sep=""), 7, 12)
-  layout(matrix(c(1,1,2,2,3,3,4,5),byrow=TRUE, ncol=2))
-  plot_distances(ani.d)
-  plot_silhouette(k, s[1,], s[2,], ds, top.n)
-  plot_clustering(ani.cl, ani.d, ani.types)
-  plot_tree(ani.ph, ani.types, ani.medoids)
-  dev.off()
-
-  # Save results
-  say("Text report")
-  write.table(ani.medoids, paste(out_base, "medoids", sep="."),
-    quote=FALSE, col.names=FALSE, row.names=FALSE)
-  save(ani.d, file=paste(out_base, "dist.rdata", sep="."))
-  classif <- cbind(names(ani.types), ani.types, ani.medoids[ ani.types ], NA)
-  ani.d.m <- 100 - as.matrix(ani.d)*100
-  for(j in 1:nrow(classif)){
-    classif[j,4] <- ani.d.m[classif[j,1], classif[j,3]]
-  }
-  write.table(classif, paste(out_base,"classif",sep="."),
-    quote=FALSE, col.names=FALSE, row.names=FALSE, sep="\t")
-
-  # Recursive search
-  say("Recursive search")
-  for(i in 1:top.n){
-    medoid <- ani.medoids[i]
-    ds_f <- names(ani.types)[ ani.types==i ]
-    say("Analyzing subclade", i, "with medoid:", medoid)
-    dir.create(paste(out_base, ".sc-", i, sep=""))
-    write.table(ds_f,
-      paste(out_base, ".sc-", i, "/miga-project.all",sep=""),
-      quote=FALSE, col.names=FALSE, row.names=FALSE)
-    if(length(ds_f) > 5){
-      a_f <- a[ (a$a %in% ds_f) & (a$b %in% ds_f), ]
-      subclades(out_base=paste(out_base, ".sc-", i, "/miga-project", sep=""),
-        thr=thr, ani=a_f)
-    }
-  }
-}
-
-#= Helper functions
-say <- function(...) { cat("[", date(), "]", ..., "\n") }
-
-generate_empty_files <- function(out_base) {
-  pdf(paste(out_base, ".pdf", sep=""), 7, 12)
-  plot(1, t="n", axes=F)
-  legend("center", "No data", bty="n")
-  dev.off()
-  file.create(paste(out_base,".1.classif",sep=""))
-  file.create(paste(out_base,".1.medoids",sep=""))
-}
-
-plot_silhouette <- function(k, s, ns, ds, top.n) {
-  # s
-  par(mar=c(4,5,1,5)+0.1)
-  plot(1, t="n", xlab="k (clusters)", ylab="", xlim=range(c(0,k)),
-    ylim=range(s), bty="n", xaxs="i", yaxt="n")
-  polygon(c(k[1], k, k[length(k)]), c(0,s,0), border=NA, col="grey80")
-  axis(2, fg="grey60", col.axis="grey60")
-  mtext("Mean silhouette", side=2, line=3, col="grey60")
-  # ns
-  par(new=TRUE)
-  plot(1, t="n", xlab="", xaxt="n", ylab="", yaxt="n", xlim=range(c(0,k)),
-    ylim=range(ns), bty="n", xaxs="i")
-  points(k, ns, type="o", pch=16, col=rgb(1/2,0,0,3/4))
-  axis(4, fg="darkred", col.axis="darkred")
-  mtext("Negative silhouette area", side=4, line=3, col="darkred")
-  # ds
-  par(new=TRUE)
-  plot(1, t="n", xlab="", xaxt="n", ylab="", yaxt="n", xlim=range(c(0,k)),
-    ylim=range(ds), bty="n", xaxs="i")
-  lines(k, ds)
-  abline(v=top.n, lty=2)
-}
-
-plot_distances <- function(dist) {
-  par(mar=c(5,4,1,2)+0.1)
-  hist(dist, border=NA, col="grey60", breaks=50, xlab="Distances", main="")
-}
-
-plot_clustering <- function(cl, dist, types) {
-  par(mar=c(5,4,4,2)+0.1)
-  top.n <- length(cl$medoids)
-  col <- ggplotColours(top.n)
-  plot(silhouette(cl), col=col)
-  if(length(labels(dist))<=15){
-    plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
-    plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
-  }else{
-    ani.mds <- cmdscale(dist, k=4)
-    if(ncol(ani.mds)==4){
-      plot(ani.mds[,1], ani.mds[,2], col=col[types], cex=1/2,
-	xlab='Component 1', ylab='Component 2')
-      plot(ani.mds[,3], ani.mds[,4], col=col[types], cex=1/2,
-	xlab='Component 3', ylab='Component 4')
-    }else{
-      plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
-      plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
-    }
-  }
-}
-
-plot_tree <- function(phy, types, medoids){
-  layout(1)
-  top.n <- length(unique(types))
-  col <- ggplotColours(top.n)
-  is.medoid <- phy$tip.label %in% medoids
-  phy$tip.label[is.medoid] <- paste(phy$tip.label[is.medoid],
-    " [", types[phy$tip.label[is.medoid]], "]", sep='')
-  plot(phy, cex=ifelse(is.medoid, 1/3, 1/6),
-    font=ifelse(is.medoid, 2, 1),
-    tip.color=col[types[phy$tip.label]])
-}
-
-ggplotColours <- function(n=6, h=c(0, 360)+15, alpha=1){
-  if ((diff(h)%%360) < 1) h[2] <- h[2] - 360/n
-  hcl(h=seq(h[1], h[2], length=n), c=100, l=65, alpha=alpha)
-}
-
-#= Main
-subclades(ani_file=argv[1], out_base=argv[2],
-  thr=ifelse(is.na(argv[3]), 1, as.numeric(argv[3])))
-