miga-base 0.3.5.1 → 0.3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 25f6a339288dbdbeda1f84f5da6b5de697d3790c0b38d138a4416ff762cae936
-  data.tar.gz: 60187223750e983fafd6088935a912d016bbf4acafaf79b548e206c50076cc04
+SHA1:
+  metadata.gz: 8b285b9906876a9f1b5366f929a4776d1689dbc1
+  data.tar.gz: 83b6843d00417fef4a8de18e4a102ad4d1899f0d
 SHA512:
-  metadata.gz: 60b59ccc8fc3bf9f5a584f3221c268839ba4cd41eb3a6c16911f1808944c20100758f74908797f13ada9d2475e4fda382c1e204b257cf6c6adcfd17db694efcf
-  data.tar.gz: e62de30119c51dc4e92e64aec954c263f84a4820f1541f014c49d93dadd8de1efb5d331c35ecbda6ccfa79c402393894b7b6ffd52edf2362b8e117fe505ff230
+  metadata.gz: acfa6eb243f7fa8985cb649ab3b701db68515e8a18c221d94cb149e51cddeec49642b6176da46956765cca8f1961aa88d71b5cb9625131cf64f9287e79e173c6
+  data.tar.gz: 4ac1f2f81854959679b53d4865efba3a36ddca14216ff9f2aef06e1c27b7b415b47c833387e73ddf3cae41369f382146f1041b9a833743d58d3aad0a954bd1ab

data/lib/miga/project/result.rb

@@ -1,8 +1,8 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require "miga/result"
-require "miga/project/base"
+require 'miga/result'
+require 'miga/project/base'
 
 ##
 # Helper module including specific functions to add project results.
@@ -31,7 +31,7 @@ module MiGA::Project::Result
   # Supported values include:
   # - +force+: A Boolean indicating if the result must be re-indexed. If true,
   #   it implies save=true.
-  def add_result(name, save=true, opts={})
+  def add_result(name, save = true, opts = {})
     return nil if @@RESULT_DIRS[name].nil?
     base = "#{path}/data/#{@@RESULT_DIRS[name]}/miga-project"
     if opts[:force]
@@ -49,21 +49,21 @@ module MiGA::Project::Result
   ##
   # Get the next distances task, saving intermediate results if +save+. Returns
   # a Symbol.
-  def next_distances(save=true) ; next_task(@@DISTANCE_TASKS, save) ; end
+  def next_distances(save = true) ; next_task(@@DISTANCE_TASKS, save) ; end
   
   ##
   # Get the next inclade task, saving intermediate results if +save+. Returns a
   # Symbol.
-  def next_inclade(save=true) ; next_task(@@INCLADE_TASKS, save) ; end
+  def next_inclade(save = true) ; next_task(@@INCLADE_TASKS, save) ; end
 
   ##
   # Get the next task from +tasks+, saving intermediate results if +save+.
   # Returns a Symbol.
-  def next_task(tasks=@@DISTANCE_TASKS+@@INCLADE_TASKS, save=true)
+  def next_task(tasks = @@DISTANCE_TASKS+@@INCLADE_TASKS, save = true)
     tasks.find do |t|
-      if metadata["run_#{t}"]==false or
+      if metadata["run_#{t}"] == false or
             (!is_clade? and @@INCLADE_TASKS.include?(t) and
-                  metadata["run_#{t}"]!=true)
+                  metadata["run_#{t}"] != true)
         false
       else
         add_result(t, save).nil?
@@ -79,10 +79,10 @@ module MiGA::Project::Result
     def add_result_distances(base)
       return nil unless result_files_exist?(base, %w[.Rdata .log .txt])
       r = MiGA::Result.new("#{base}.json")
-      r.add_file(:rdata, "miga-project.Rdata")
-      r.add_file(:matrix, "miga-project.txt")
-      r.add_file(:log, "miga-project.log")
-      r.add_file(:hist, "miga-project.hist")
+      r.add_file(:rdata,  'miga-project.Rdata')
+      r.add_file(:matrix, 'miga-project.txt')
+      r.add_file(:log,    'miga-project.log')
+      r.add_file(:hist,   'miga-project.hist')
       r
     end
 
@@ -91,10 +91,11 @@ module MiGA::Project::Result
       return nil unless is_clade? or result_files_exist?(base,
         %w[.pdf .classif .medoids .class.tsv .class.nwk])
       r = add_result_iter_clades(base)
-      r.add_file(:aai_tree,	"miga-project.aai.nwk")
-      r.add_file(:proposal,	"miga-project.proposed-clades")
-      r.add_file(:clades_aai90,	"miga-project.aai90-clades")
-      r.add_file(:clades_ani95,	"miga-project.ani95-clades")
+      r.add_file(:aai_tree,      'miga-project.aai.nwk')
+      r.add_file(:proposal,      'miga-project.proposed-clades')
+      r.add_file(:clades_aai90,  'miga-project.aai90-clades')
+      r.add_file(:clades_ani95,  'miga-project.ani95-clades')
+      r.add_file(:medoids_ani95, 'miga-project.ani95-medoids')
       r
     end
 
@@ -102,28 +103,28 @@ module MiGA::Project::Result
       return nil unless result_files_exist?(base,
         %w[.pdf .classif .medoids .class.tsv .class.nwk])
       r = add_result_iter_clades(base)
-      r.add_file(:ani_tree, "miga-project.ani.nwk")
+      r.add_file(:ani_tree, 'miga-project.ani.nwk')
       r
     end
 
     def add_result_iter_clades(base)
       r = MiGA::Result.new("#{base}.json")
-      r.add_file(:report,	"miga-project.pdf")
-      r.add_file(:class_table,	"miga-project.class.tsv")
-      r.add_file(:class_tree,	"miga-project.class.nwk")
-      r.add_file(:classif,	"miga-project.classif")
-      r.add_file(:medoids,	"miga-project.medoids")
+      r.add_file(:report,      'miga-project.pdf')
+      r.add_file(:class_table, 'miga-project.class.tsv')
+      r.add_file(:class_tree,  'miga-project.class.nwk')
+      r.add_file(:classif,     'miga-project.classif')
+      r.add_file(:medoids,     'miga-project.medoids')
       r
     end
 
     def add_result_ogs(base)
       return nil unless result_files_exist?(base, %w[.ogs .stats])
       r = MiGA::Result.new("#{base}.json")
-      r.add_file(:ogs, "miga-project.ogs")
-      r.add_file(:abc, "miga-project.abc")
-      r.add_file(:stats, "miga-project.stats")
-      r.add_file(:core_pan, "miga-project.core-pan.tsv")
-      r.add_file(:core_pan_plot, "miga-project.core-pan.pdf")
+      r.add_file(:ogs,   'miga-project.ogs')
+      r.add_file(:abc,   'miga-project.abc')
+      r.add_file(:stats, 'miga-project.stats')
+      r.add_file(:core_pan,      'miga-project.core-pan.tsv')
+      r.add_file(:core_pan_plot, 'miga-project.core-pan.pdf')
       r
     end
 
@@ -131,8 +132,8 @@ module MiGA::Project::Result
       return nil unless
         result_files_exist?(base, %w[.taxonomy.json .metadata.db])
       r = MiGA::Result.new("#{base}.json")
-      r.add_file(:taxonomy_index, "miga-project.taxonomy.json")
-      r.add_file(:metadata_index, "miga-project.metadata.db")
+      r.add_file(:taxonomy_index, 'miga-project.taxonomy.json')
+      r.add_file(:metadata_index, 'miga-project.metadata.db')
       r
     end
 

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 5, 1]
+  VERSION = [0.3, 6, 0]
   
   ##
   # Nickname for the current major.minor version.

data/utils/distance/commands.rb

@@ -18,6 +18,7 @@ module MiGA::DistanceRunner::Commands
           dataset.name, target.name, tmp_dbs[:aai]).tap{ checkpoint :aai }
   end
 
+  ##
   # Estimates AAI against +target+ using hAAI
   def haai(target)
     haai = aai_cmd(tmp_file("ess_genes.fa"),
@@ -34,6 +35,7 @@ module MiGA::DistanceRunner::Commands
     aai
   end
 
+  ##
   # Calculates ANI against +target+
   def ani(target)
     # Check if the request makes sense
@@ -49,6 +51,15 @@ module MiGA::DistanceRunner::Commands
         dataset.name, target.name, tmp_dbs[:ani]).tap{ checkpoint :ani }
   end
 
+  ##
+  # Calculates and returns ANI against +target+ if AAI >= 85%. Returns
+  # +nil+ otherwise
+  def ani_after_aai(target)
+    aai = aai(target)
+    ani(target) unless aai.nil? or aai < 85.0
+  end
+
+  ##
   # Execute an AAI command
   def aai_cmd(f1, f2, n1, n2, db, o={})
     o = opts.merge(o)
@@ -59,6 +70,7 @@ module MiGA::DistanceRunner::Commands
     (v.nil? or v.empty?) ? 0 : v.to_f
   end
 
+  ##
   # Execute an ANI command
   def ani_cmd(f1, f2, n1, n2, db, o={})
     o = opts.merge(o)

data/utils/distance/database.rb

@@ -2,6 +2,7 @@
 require 'sqlite3'
 
 module MiGA::DistanceRunner::Database
+  ##
   # Check for corrupt files and create empty databases
   def initialize_dbs!(for_ref)
     @dbs = {}
@@ -33,6 +34,7 @@ module MiGA::DistanceRunner::Database
     end
   end
 
+  ##
   # Path to the database +metric+ for +dataset_name+ in +project+
   # (assumes that +dataset_name+ is a reference dataset)
   def ref_db(metric, dataset_name=nil)
@@ -48,35 +50,66 @@ module MiGA::DistanceRunner::Database
     File.expand_path(b, home)
   end
 
+  ##
   # Path to the database +metric+ for +dataset+ (assumes that +dataset+ is a
   # query dataset)
   def query_db(metric)
     File.expand_path("#{dataset.name}.#{metric}.db", home)
   end
 
+  ##
   # Get the stored +metric+ value against +target+
   def stored_value(target, metric)
     # Check if self.dataset -> target is done (previous run)
     y = value_from_db(dataset.name, target.name, tmp_dbs[metric], metric)
     return y unless y.nil? or y.zero?
+
     # Check if self.dataset <- target is done (another thread)
-    if dataset.is_ref? and project.path==ref_project.path
-      y = value_from_db(target.name, dataset.name, ref_db(metric, target.name), metric)
-      return y unless y.nil? or y.zero?
+    if dataset.is_ref? and project.path == ref_project.path
+      y = data_from_db(
+        target.name, dataset.name, ref_db(metric, target.name), metric)
+      unless y.nil? or y.first.zero?
+        # Store a copy
+        data_to_db(dataset.name, target.name, tmp_dbs[metric], metric, y)
+        return y.first
+      end
     end
     nil
   end
 
+  ##
   # Get the value of +metric+ in the +db+ database between +n1+ and +n2+
   def value_from_db(n1, n2, db, metric)
+    y = data_from_db(n1, n2, db, metric)
+    y.first unless y.nil?
+  end
+
+  ##
+  # Get the +metric+ data in the +db+ database between +n1+ and +n2+. Returns an
+  # Array with the metric, standard deviation, number of matches, and maximum
+  # possible number of matches
+  def data_from_db(n1, n2, db, metric)
     y = nil
     SQLite3::Database.new(db) do |conn|
-      y = conn.execute("select #{metric} from #{metric} where seq1=? and seq2=?", [n1, n2]).first
-      y = y.first unless y.nil?
+      y = conn.execute(
+        "select #{metric}, sd, n, omega from #{metric} where seq1=? and seq2=?",
+        [n1, n2]).first
     end if File.size? db
     y
   end
 
+  ##
+  # Save +data+ of +metric+ between +n1+ and +n2+ in the +db+ database.
+  def data_to_db(n1, n2, db, metric, data)
+    SQLite3::Database.new(db) do |conn|
+      conn.execute(
+        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+        "values (?, ?, ?, ?, ?, ?)", [n1, n2] + data)
+    end
+    checkpoint metric
+  end
+
+  ##
   # Iterates for each entry in +db+
   def foreach_in_db(db, metric, &blk)
     SQLite3::Database.new(db) do |conn|

data/utils/distance/runner.rb

@@ -30,7 +30,9 @@ class MiGA::DistanceRunner
     if opts[:run_taxonomy] && project.metadata[:ref_project]
       @home = File.expand_path('05.taxonomy', @home)
       @ref_project = MiGA::Project.load(project.metadata[:ref_project])
-      raise "Cannot load reference project: #{project.metadata[:ref_project]}" if @ref_project.nil?
+      if @ref_project.nil?
+        raise "Cannot load reference project: #{project.metadata[:ref_project]}"
+      end
     else
       @ref_project = project
     end
@@ -55,47 +57,61 @@ class MiGA::DistanceRunner
   def go_ref!
     # Initialize databases
     initialize_dbs! true
+
     # first-come-first-serve traverse
     ref_project.each_dataset do |ds|
       next if !ds.is_ref? or ds.is_multi? or ds.result(:essential_genes).nil?
       puts "[ #{Time.now} ] #{ds.name}"
-      aai = aai(ds)
-      ani(ds) unless aai.nil? or aai < 90.0
+      ani_after_aai(ds)
     end
+
     # Finalize
     [:haai, :aai, :ani].each{ |m| checkpoint! m if db_counts[m] > 0 }
   end
 
+  ##
   # Launch analysis for query datasets
   def go_query!
     # Check if project is ready
-    v = ref_project.is_clade? ? [:subclades, :ani] : [:clade_finding, :aai]
-    res = ref_project.result(v[0])
+    tsk = ref_project.is_clade? ? [:subclades, :ani] : [:clade_finding, :aai]
+    res = ref_project.result(tsk[0])
     return if res.nil?
+
     # Initialize the databases
     initialize_dbs! false
     # Calculate the classification-informed AAI/ANI traverse
-    results = File.expand_path("#{dataset.name}.#{v[1]}-medoids.tsv", home)
-    fh = File.open(results, "w")
-    classif, val_cls = *classify(res.dir, ".", v[1], fh)
+    results = File.expand_path("#{dataset.name}.#{tsk[1]}-medoids.tsv", home)
+    fh = File.open(results, 'w')
+    classif, val_cls = *classify(res.dir, '.', tsk[1], fh)
     fh.close
+
     # Calculate all the AAIs/ANIs against the lowest subclade (if classified)
     par_dir = File.dirname(File.expand_path(classif, res.dir))
-    par = File.expand_path("miga-project.classif", par_dir)
+    par = File.expand_path('miga-project.classif', par_dir)
+    closest = {dataset: nil, ani: 0.0}
     if File.size? par
-      File.open(par, "r") do |fh|
+      File.open(par, 'r') do |fh|
         fh.each_line do |ln|
           r = ln.chomp.split("\t")
-          next unless r[1].to_i==val_cls
-          target = ref_project.dataset(r[0])
-          aai = (v[1]==:aai) ? aai(target) : 100.0
-          ani(target) if aai >= 90.0
+          next unless r[1].to_i == val_cls
+          ani = ani_after_aai(ref_project.dataset(r[0]))
+          closest = {ds: r[0], ani: ani} unless ani.nil? or ani < closest[:ani]
         end
       end
     end
+
+    # Calculate all the AAIs/ANIs against the closest ANI95-clade (if ANI > 95%)
+    cl_path = File.expand_path('miga-project.ani95-clades', home)
+    if File.size? cl_path and tsk[0] == :clade_finding and closest[:ani] >= 95.0
+      File.foreach(cl_path).
+        map  { |i| i.chomp.split(',') }.
+        find { |i| i.include? closest[:ds] }.
+        each { |i| ani_after_aai(ref_project.dataset(i)) }
+    end
+
     # Finalize
     [:haai, :aai, :ani].each{ |m| checkpoint! m if db_counts[m] > 0 }
-    build_medoids_tree(v[1])
+    build_medoids_tree(tsk[1])
     transfer_taxonomy(tax_test)
   end
 

data/utils/enveomics/Pipelines/assembly.pbs/FastA.N50.pl

@@ -0,0 +1,56 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Oct 07 2015
+# @license: artistic license 2.0
+#
+use strict;
+use warnings;
+use List::Util qw/sum min max/;
+
+my ($seqs, $minlen, $n__) = @ARGV;
+$seqs or die "
+Description:
+   Calculates the N50 value of a set of sequences.  Alternatively, it
+   can calculate other N** values.  It also calculates the total number
+   of sequences and the total added length.
+   
+Usage:
+   $0 seqs.fa[ minlen[ **]]
+
+   seqs.fa	A FastA file containing the sequences.
+   minlen	(optional) The minimum length to take into consideration.
+   		By default: 0.
+   **		Value N** to calculate.  By default: 50 (N50).
+";
+$minlen ||= 0;
+$n__    ||= 50;
+
+my @len = ();
+open SEQ, "<", $seqs or die "Cannot open file: $seqs: $!\n";
+while(<SEQ>){
+   if(/^>/){
+      push @len, 0;
+   }else{
+      next if /^;/;
+      chomp;
+      s/\W//g;
+      $len[-1]+=length $_;
+   }
+}
+close SEQ;
+@len = sort { $a <=> $b } map { $_>=$minlen?$_:() } @len;
+my $tot = (sum(@len) || 0);
+
+my $thr = $n__*$tot/100;
+my $pos = 0;
+for(@len){
+   $pos+= $_;
+   if($pos>=$thr){
+      print "N$n__: $_\n";
+      last;
+   }
+}
+print "Sequences: ".scalar(@len)."\n";
+print "Total length: $tot\n";
+

data/utils/enveomics/Pipelines/assembly.pbs/FastA.filterN.pl

@@ -0,0 +1,60 @@
+#!/usr/bin/env perl
+# 
+# @author  Luis M. Rodriguez-R
+# @update  Oct-07-2015
+# @license artistic license 2.0
+# 
+
+use warnings;
+use strict;
+
+my($file, $content, $stretch) = @ARGV;
+$file or die <<HELP
+
+Description:
+   Filter sequences by N-content and presence of long homopolymers.
+Usage:
+   $0 sequences.fa [content [stretch]] > filtered.fa
+Where:
+   sequences.fa	Input file in FastA format
+   content	A number between 0 and 1 indicating the maximum proportion of Ns
+   		(1 to turn off, 0.5 by default)
+   stretch	A number indicating the maximum number of consecutive identical
+   		nucleotides allowed (0 to turn off, 100 by default)
+   filtered.fa	Filtered set of sequences.
+
+HELP
+;
+($content ||= 0.5)+=0;
+($stretch ||= 100)+=0;
+
+my $good = 0;
+my $N = 0;
+
+FASTA: {
+   local $/ = "\n>";
+   open FILE, "<", $file or die "I can not open the file: $file: $!\n";
+   SEQ: while(<FILE>){
+      $N++;
+      s/^;.*//gm;
+      s/>//g;
+      my($n,$s) = split /\n/, $_, 2;
+      (my $clean = $s) =~ s/[^ACTGN]//g;
+      if($content < 1){
+         (my $Ns = $clean) =~ s/[^N]//g;
+	 next SEQ if length($Ns)>length($clean)*$content;
+      }
+      if($stretch > 0){
+         for my $nuc (qw(A C T G N)){
+	    next SEQ if $clean =~ m/[$nuc]{$stretch}/;
+	 }
+      }
+      print ">$n\n$s\n";
+      $good++;
+   }
+   close FILE;
+   print STDERR "Total sequences: $N\nAfter filtering: $good\n";
+}
+
+
+

data/utils/enveomics/Pipelines/assembly.pbs/FastA.length.pl

@@ -0,0 +1,38 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M Rodriguez-R
+# @update  Oct-07-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+
+$#ARGV>=0 or die "
+Usage:
+   $0 seqs.fa... > length.txt
+
+   seqs.fa	One or more FastA files.
+   length.txt	A table with the lengths of the sequences.
+
+";
+
+for my $fa (@ARGV){
+   open FA, "<", $fa or die "Cannot open file: $fa: $!\n";
+   my $def = '';
+   my $len = 0;
+   while(<FA>){
+      next if /^;/;
+      if(m/^>(\S+)\s?/){
+         print "$def\t$len\n" if $def;
+	 $def = $1;
+	 $len = 0;
+      }else{
+         s/[^A-Za-z]//g;
+	 $len+= length $_;
+      }
+   }
+   print "$def\t$len\n" if $def;
+   close FA;
+}
+

data/utils/enveomics/Pipelines/blast.pbs/FastA.split.pl

@@ -0,0 +1,55 @@
+#!/usr/bin/env perl
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update Oct-13-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Symbol;
+
+my ($file, $base, $outN) = @ARGV;
+
+$outN ||= 12;
+($file and $base) or die "
+Usage
+   $0 in_file.fa out_base[ no_files]
+   
+   in_file.fa	Input file in FastA format.
+   out_base	Prefix for the name of the output files.  It will
+   		be appended with .<i>.fa, where <i> is a consecutive
+		number starting in 1.
+   no_files	Number of files to generate.  By default: 12.
+
+";
+
+
+my @outSym = ();
+for my $i (1 .. $outN){
+   $outSym[$i-1] = gensym;
+   open $outSym[$i-1], ">", "$base.$i.fa" or
+      die "I can not create the file: $base.$i.fa: $!\n";
+}
+
+
+my($i, $seq) = (-1, '');
+open FILE, "<", $file or die "I can not read the file: $file: $!\n";
+while(my $ln=<FILE>){
+   next if $ln=~/^;/;
+   if($ln =~ m/^>/){
+      print { $outSym[$i % $outN] } $seq if $seq;
+      $i++;
+      $seq = '';
+   }
+   $seq.=$ln;
+}
+print { $outSym[$i % $outN] } $seq if $seq;
+close FILE;
+
+for(my $j=0; $j<$outN; $j++){
+   close $outSym[$j];
+}
+
+print STDERR "Sequences: ".($i+1)."\nFiles: $outN\n";
+

data/utils/find-medoid.R

@@ -0,0 +1,31 @@
+#!/usr/bin/env Rscript
+#
+# @package MiGA
+# @license Artistic-2.0
+#
+
+#= Load stuff
+argv <- commandArgs(trailingOnly=T)
+suppressPackageStartupMessages(library(ape))
+
+find_medoids <- function(dist, out, clades) {
+  dist <- as.matrix(dist)
+  cl <- read.table(clades, header = FALSE, sep = '\t', as.is = TRUE)[,1]
+  medoids <- c()
+  for(i in cl){
+    lab <- strsplit(i, ',')[[1]]
+    cat('Clade of:', lab[1], '\n')
+    if(length(lab) == 1) {
+      med <- lab
+    } else {
+      med <- lab[which.min(colSums(dist[lab, lab], na.rm = TRUE))]
+    }
+    medoids <- c(medoids, med)
+  }
+  write.table(medoids, out, quote = FALSE, row.names = FALSE, col.names = FALSE)
+}
+
+#= Main
+load(argv[1])
+find_medoids(dist = ani.d, out = argv[2], clades = argv[3])
+

data/utils/representatives.rb

@@ -6,15 +6,18 @@ require 'miga'
 proj_path = ARGV.shift or raise "Usage: #{$0} path/to/project"
 
 # Load MiGA object
-p = MiGA::Project.load(proj_path) or raise "Cannot load project: #{proj_path}"
-pr = p.result(:clade_finding) or raise "Unavailable result: clade_finding"
-pf = pr.file_path(:clades_ani95) or raise "Unavailable result file: proposal"
+p = MiGA::Project.load(proj_path) or
+  raise "Cannot load project: #{proj_path}"
+pr = p.result(:clade_finding) or
+  raise "Unavailable result: clade_finding"
+pf = pr.file_path(:clades_ani95) or
+  raise "Unavailable result file: clades_ani95"
 
 # Read ANIspp
 ani_spp = []
 File.open(pf, 'r') do |fh|
   fh.each_line do |ln|
-    next if $.==1
+    next if $.==1 and ln.chomp == 'G' # <- Legacy check
     ani_spp << ln.chomp.split(',')
   end
 end

data/utils/subclade/pipeline.rb

@@ -2,7 +2,7 @@
 # High-end pipelines for SubcladeRunner
 module MiGA::SubcladeRunner::Pipeline
 
-  # Run species-level clusterings using ANI>95% / AAI>90%
+  # Run species-level clusterings using ANI > 95% / AAI > 90%
   def cluster_species
     tasks = {ani95: [:ani_distances, 95.0], aai90: [:aai_distances, 90.0]}
     tasks.each do |k, par|
@@ -23,8 +23,20 @@ module MiGA::SubcladeRunner::Pipeline
       end
       ofh.close
       # Cluster genomes
-      `ogs.mcl.rb -o '#{ogs_file}' --abc '#{abc_path}' -t '#{opts[:thr]}'`
+      `ogs.mcl.rb -o '#{ogs_file}.tmp' --abc '#{abc_path}' -t '#{opts[:thr]}'`
+      File.open(ogs_file, 'w') do |fh|
+        File.foreach("#{ogs_file}.tmp").with_index do |ln, lno|
+          fh.puts ln if lno != 0
+        end
+      end
+      File.unlink "#{ogs_file}.tmp"
     end
+
+    # Find species medoids
+    src = File.expand_path('utils/find-medoid.R', MiGA::MiGA.root_path)
+    `Rscript '#{src}' miga-project.dist.rdata \
+      miga-project.ani95-medoids miga-project.ani95-clades`
+
     # Propose clades
     ofh = File.open('miga-project.proposed-clades', 'w')
     File.open('miga-project.ani95-clades', 'r') do |ifh|
@@ -42,7 +54,8 @@ module MiGA::SubcladeRunner::Pipeline
     step = :"#{metric}_distances"
     metric_res = project.result(step) or raise "Incomplete step #{step}"
     matrix = metric_res.file_path(:matrix)
-    `Rscript '#{src}' '#{matrix}' miga-project '#{opts[:thr]}'`
+    `Rscript '#{src}' '#{matrix}' miga-project '#{opts[:thr]}' \
+      miga-project.ani95-medoids`
     File.rename('miga-project.nwk',"miga-project.#{metric}.nwk") if
           File.exist? 'miga-project.nwk'
   end

data/utils/subclade/runner.rb

@@ -3,7 +3,6 @@ require_relative 'base.rb'
 require_relative 'temporal.rb'
 require_relative 'pipeline.rb'
 
-
 class MiGA::SubcladeRunner
 
   include MiGA::SubcladeRunner::Temporal
@@ -17,7 +16,7 @@ class MiGA::SubcladeRunner
           raise "No project at #{project_path}"
     @step = step.to_sym
     clades_dir = File.expand_path('data/10.clades', project.path)
-    @home = File.expand_path(@step==:clade_finding ? '01.find' : '02.ani',
+    @home = File.expand_path(@step == :clade_finding ? '01.find' : '02.ani',
           clades_dir)
     @opts[:thr] ||= ENV.fetch("CORES"){ 2 }.to_i
   end
@@ -29,7 +28,7 @@ class MiGA::SubcladeRunner
     Dir.mktmpdir do |tmp_dir|
       @tmp = tmp_dir
       create_temporals
-      step==:clade_finding ? go_clade_finding! : go_subclades!
+      step == :clade_finding ? go_clade_finding! : go_subclades!
     end
   end
 

data/utils/subclades.R

@@ -5,7 +5,7 @@
 #
 
 #= Load stuff
-argv <- commandArgs(trailingOnly=T)
+argv <- commandArgs(trailingOnly = TRUE)
 suppressPackageStartupMessages(library(ape))
 suppressPackageStartupMessages(library(vegan))
 suppressPackageStartupMessages(library(cluster))
@@ -13,38 +13,44 @@ suppressPackageStartupMessages(library(parallel))
 suppressPackageStartupMessages(library(enveomics.R))
 
 #= Main function
-subclades <- function(ani_file, out_base, thr=1, ani.d=dist(0)) {
-  say("==> Out base:", out_base, "<==")
+subclades <- function(ani_file, out_base, thr = 1, ani.d = dist(0), sel = NA) {
+  say('==> Out base:', out_base, '<==')
 
   # Normalize input matrix
-  dist_rdata = paste(out_base, "dist.rdata", sep=".")
+  dist_rdata = paste(out_base, 'dist.rdata', sep = '.')
   if(!missing(ani_file)){
-    if(length(ani.d)==0 && !file.exists(dist_rdata)){
+    if(length(ani.d) == 0 && !file.exists(dist_rdata)){
       # Read from ani_file
-      a <- read.table(gzfile(ani_file), sep="\t", header=TRUE, as.is=TRUE)
+      a <- read.table(gzfile(ani_file), sep = '\t', header = TRUE, as.is = TRUE)
       if(nrow(a)==0){
         generate_empty_files(out_base)
         return(NULL)
       }
-      say("Distances")
+      if(!is.na(sel) and file.exists(sel)){
+        say('Filter selection')
+        lab <- read.table(sel, sep='\t', head=FALSE, as.is=TRUE)[,1]
+        a <- a[a$a %in% lab & a$b %in% lab, ]
+      }
+      say('Distances')
       a$d <- 1 - (a$value/100)
-      ani.d <- enve.df2dist(a, 'a', 'b', 'd', default.d=max(a$d)*1.2)
-      save(ani.d, file=dist_rdata)
+      ani.d <- enve.df2dist(a, 'a', 'b', 'd', default.d = max(a$d)*1.2)
+      save(ani.d, file = dist_rdata)
     }
   }
 
   # Read result if the subclade is ready, run it otherwise
-  if(file.exists(paste(out_base,"classif",sep="."))){
+  if(file.exists(paste(out_base, 'classif', sep = '.'))){
     say("Loading")
-    ani.medoids <- read.table(paste(out_base, "medoids", sep="."),
-          sep=' ', as.is=TRUE)[,1]
-    a <- read.table(paste(out_base,"classif",sep="."), sep="\t", as.is=TRUE)
+    ani.medoids <- read.table(paste(out_base, "medoids", sep = "."),
+      sep = ' ', as.is = TRUE)[,1]
+    a <- read.table(paste(out_base, "classif", sep="."),
+      sep = '\t', as.is = TRUE)
     ani.types <- a[,2]
     names(ani.types) <- a[,1]
-    if(length(ani.d)==0) load(dist_rdata)
+    if(length(ani.d) == 0) load(dist_rdata)
   }else{
     res <- subclade_clustering(out_base, thr, ani.d, dist_rdata)
-    if(length(res)==0) return(NULL)
+    if(length(res) == 0) return(NULL)
     ani.medoids <- res[['ani.medoids']]
     ani.types <- res[['ani.types']]
     ani.d <- res[['ani.d']]
@@ -230,7 +236,7 @@ ggplotColours <- function(n=6, h=c(0, 360)+15, alpha=1){
 }
 
 #= Main
-options(warn=1)
-subclades(ani_file=argv[1], out_base=argv[2],
-  thr=ifelse(is.na(argv[3]), 1, as.numeric(argv[3])))
+options(warn = 1)
+subclades(ani_file = argv[1], out_base = argv[2],
+  thr = ifelse(is.na(argv[3]), 1, as.numeric(argv[3])), sel = argv[4])
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.5.1
+  version: 0.3.6.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-18 00:00:00.000000000 Z
+date: 2018-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -341,7 +341,6 @@ files:
 - utils/enveomics/Scripts/gi2tax.rb
 - utils/enveomics/Scripts/in_silico_GA_GI.pl
 - utils/enveomics/Scripts/lib/data/essential.hmm.gz
-- utils/enveomics/Scripts/lib/enveomics.R
 - utils/enveomics/Scripts/lib/enveomics_rb/enveomics.rb
 - utils/enveomics/Scripts/lib/enveomics_rb/jplace.rb
 - utils/enveomics/Scripts/lib/enveomics_rb/og.rb
@@ -454,6 +453,7 @@ files:
 - utils/enveomics/enveomics.R/man/z$-methods.Rd
 - utils/enveomics/globals.mk
 - utils/enveomics/manifest.json
+- utils/find-medoid.R
 - utils/index_metadata.rb
 - utils/mytaxa_scan.R
 - utils/mytaxa_scan.rb
@@ -495,7 +495,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: MiGA

data/utils/enveomics/Pipelines/assembly.pbs/FastA.N50.pl

@@ -1 +0,0 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.N50.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.filterN.pl

@@ -1 +0,0 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.filterN.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.length.pl

@@ -1 +0,0 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.length.pl

data/utils/enveomics/Pipelines/blast.pbs/FastA.split.pl

@@ -1 +0,0 @@
-utils/enveomics/Pipelines/blast.pbs/../../Scripts/FastA.split.pl

data/utils/enveomics/Scripts/lib/enveomics.R

@@ -1 +0,0 @@
-utils/enveomics/Scripts/lib/../../enveomics.R