miga-base 1.1.2.4 → 1.1.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: beb13a42d514a4cf731b3b042096728cd6d615124f1412046ad16fb8de2b709e
-  data.tar.gz: 64e32095ac59863d4005e602e55a1396530aac12da40fe88316985c95a90eb12
+  metadata.gz: 683a93a244b106eb24f94cff47695b28cc051701ecd4ea83b289a6650fdf3985
+  data.tar.gz: 6ce192df7a66502535a5c24406d9d3f1d0a1d51c11f3fb6e08f353547661c053
 SHA512:
-  metadata.gz: d8f813d3b32cfdfb86da746e63c8e7a3a5c142822dc44a7c2588219368bed31851a77cd05ce30ee2d1dc02d930beaab836cda58a122cc5955d6ec0235b2edb42
-  data.tar.gz: 81d773c3d4078ec3fd27d389bb1159aa9585dafdd2efdb5a917471ec784d9ebf24e67f2fba5358a4c4c589d03a12ab9f3e1b120e83390e0f0354a059a0cbb568
+  metadata.gz: cd07ba5ea8d3088ba9e7752c13bcdc07a0cf73175687c021e93c6df9361abd6dab361ffaed562f421f551999deee3de4f32d388f96f050ff45750d22bbe3eaf3
+  data.tar.gz: 18ba878127bc0abb7aed7b63398091cc2ed0fbee0d8629850a25d26ab9690f4489a8aca4fa8a3e2f2a8d4e80bdf1fd065511c23774327e988fac9ff0df645041

data/lib/miga/cli/action/doctor/base.rb

@@ -59,13 +59,14 @@ module MiGA::Cli::Action::Doctor::Base
         next if (lineno += 1) == 1
 
         r = ln.split("\t")
-        next unless [1, 2].map { |i| p.dataset(r[i]).nil? }.any?
+        names = [r[0], r[1]]
+        next unless names.any? { |i| p.dataset(i).nil? }
 
-        [1, 2].each do |i|
-          if p.dataset(r[i]).nil? || !p.dataset(r[i]).active?
-            notok[r[i]] = true
+        names.each do |i|
+          if p.dataset(i).nil? || !p.dataset(i).active?
+            notok[i] = true
           else
-            fix[r[i]] = true
+            fix[i] = true
           end
         end
       end

data/lib/miga/project/result.rb

@@ -55,10 +55,11 @@ module MiGA::Project::Result
   ##
   # Add result of any type +:*_distances+ at +base+ (no +_opts+ supported).
   def add_result_distances(base, _opts)
-    return nil unless result_files_exist?(base, %w[.Rdata .txt])
+    return nil unless result_files_exist?(base, %w[.rds .txt])
 
     r = MiGA::Result.new("#{base}.json")
-    r.add_file(:rdata,  'miga-project.Rdata')
+    r.add_file(:rds,    'miga-project.rds')
+    r.add_file(:rdata,  'miga-project.Rdata') # Legacy file
     r.add_file(:matrix, 'miga-project.txt')
     r.add_file(:log,    'miga-project.log') # Legacy file
     r.add_file(:hist,   'miga-project.hist')
@@ -82,12 +83,13 @@ module MiGA::Project::Result
     end
 
     r = add_result_iter_clades(base)
-    r.add_file(:aai_tree,      'miga-project.aai.nwk')
-    r.add_file(:proposal,      'miga-project.proposed-clades')
-    r.add_file(:clades_aai90,  'miga-project.aai90-clades')
-    r.add_file(:clades_ani95,  'miga-project.ani95-clades')
-    r.add_file(:clades_gsp,  'miga-project.gsp-clades')
-    r.add_file(:medoids_gsp, 'miga-project.gsp-medoids')
+    r.add_file(:aai_dist_rds, 'miga-project.dist.rds')
+    r.add_file(:aai_tree,     'miga-project.aai.nwk')
+    r.add_file(:proposal,     'miga-project.proposed-clades')
+    r.add_file(:clades_aai90, 'miga-project.aai90-clades')
+    r.add_file(:clades_ani95, 'miga-project.ani95-clades')
+    r.add_file(:clades_gsp,   'miga-project.gsp-clades')
+    r.add_file(:medoids_gsp,  'miga-project.gsp-medoids')
     r
   end
 
@@ -105,6 +107,7 @@ module MiGA::Project::Result
 
     r = add_result_iter_clades(base)
     r.add_file(:ani_tree, 'miga-project.ani.nwk')
+    r.add_file(:ani_dist_rds, 'miga-project.dist.rds')
     r
   end
 

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.1, 2, 4].freeze
+  VERSION = [1.1, 3, 3].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2021, 11, 16)
+  VERSION_DATE = Date.new(2021, 11, 29)
 
   ##
   # References of MiGA

data/scripts/aai_distances.bash

@@ -22,15 +22,15 @@ DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
 
 # R-ify
 cat <<R | R --vanilla
-file <- gzfile('miga-project.txt.gz')
-aai <- read.table(file, sep = '\t', header = TRUE, as.is = TRUE)
-save(aai, file = 'miga-project.Rdata')
-if(sum(aai[, 'a'] != aai[, 'b']) > 0) {
-  h <- hist(aai[aai[, 'a'] != aai[, 'b'], 'value'], breaks = 100, plot = FALSE)
-  len <- length(h[['breaks']])
+file <- gzfile("miga-project.txt.gz")
+aai <- read.table(file, sep = "\t", header = TRUE, as.is = TRUE)
+saveRDS(aai, file = "miga-project.rds")
+if(sum(aai[, "a"] != aai[, "b"]) > 0) {
+  h <- hist(aai[aai[, "a"] != aai[, "b"], "value"], breaks = 100, plot = FALSE)
+  len <- length(h[["breaks"]])
   write.table(
-    cbind(h[['breaks']][-len], h[['breaks']][-1], h[['counts']]),
-    file = 'miga-project.hist', quote = FALSE, sep = '\t',
+    cbind(h[["breaks"]][-len], h[["breaks"]][-1], h[["counts"]]),
+    file = "miga-project.hist", quote = FALSE, sep = "\t",
     col.names = FALSE, row.names = FALSE
   )
 }

data/scripts/ani_distances.bash

@@ -22,15 +22,15 @@ DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
 
 # R-ify
 cat <<R | R --vanilla
-file <- gzfile('miga-project.txt.gz')
-ani <- read.table(file, sep = '\t', header = TRUE, as.is = TRUE)
-save(ani, file = 'miga-project.Rdata')
-if(sum(ani[, 'a'] != ani[, 'b']) > 0) {
-  h <- hist(ani[ani[, 'a'] != ani[, 'b'], 'value'], breaks = 100, plot = FALSE)
-  len <- length(h[['breaks']])
+file <- gzfile("miga-project.txt.gz")
+ani <- read.table(file, sep = "\t", header = TRUE, as.is = TRUE)
+saveRDS(ani, file = "miga-project.rds")
+if(sum(ani[, "a"] != ani[, "b"]) > 0) {
+  h <- hist(ani[ani[, "a"] != ani[, "b"], "value"], breaks = 100, plot = FALSE)
+  len <- length(h[["breaks"]])
   write.table(
-    cbind(h[['breaks']][-len], h[['breaks']][-1], h[['counts']]),
-    file = 'miga-project.hist', quote = FALSE, sep = '\t',
+    cbind(h[["breaks"]][-len], h[["breaks"]][-1], h[["counts"]]),
+    file = "miga-project.hist", quote = FALSE, sep = "\t",
     col.names = FALSE, row.names = FALSE
   )
 }

data/scripts/haai_distances.bash

@@ -15,7 +15,7 @@ ruby -I "$MIGA/lib" "$MIGA/utils/cleanup-databases.rb" "$PROJECT" "$CORES"
 # No real need for hAAI distributions at all
 echo -n "" > miga-project.log
 echo -n "" > miga-project.txt
-echo "aai <- NULL; save(aai, file = 'miga-project.Rdata')" | R --vanilla
+echo 'aai <- NULL; saveRDS(aai, file = "miga-project.rds")' | R --vanilla
 
 # Finalize
 miga_end_project_step "$DIR"

data/test/project_test.rb

@@ -82,7 +82,7 @@ class ProjectTest < Test::Unit::TestCase
   def test_add_result
     p1 = project
     assert_nil(p1.add_result(:doom))
-    %w[.Rdata .log .txt .done].each do |x|
+    %w[.rds .log .txt .done].each do |x|
       assert_nil(p1.add_result(:haai_distances))
       FileUtils.touch(
         File.join(
@@ -117,11 +117,12 @@ class ProjectTest < Test::Unit::TestCase
     # Project tasks
     expected_files = {
       project_stats: %w[.taxonomy.json .metadata.db],
-      haai_distances: %w[.Rdata .log .txt],
-      aai_distances: %w[.Rdata .log .txt],
-      ani_distances: %w[.Rdata .log .txt],
-      clade_finding: %w[.pdf .classif .medoids
-                        .class.tsv .class.nwk .proposed-clades],
+      haai_distances: %w[.rds .log .txt],
+      aai_distances: %w[.rds .log .txt],
+      ani_distances: %w[.rds .log .txt],
+      clade_finding: %w[
+        .pdf .classif .medoids .class.tsv .class.nwk .proposed-clades
+      ],
       subclades: %w[.pdf .classif .medoids .class.tsv .class.nwk],
       ogs: %w[.ogs .stats]
     }

data/utils/FastAAI/FastAAI

@@ -2702,6 +2702,8 @@ def merge_db_opts():
 			
 	parser.add_argument('-d', '--donors',      dest = 'donors',    default = None,   help =  'Comma-separated string of paths to one or more donor databases. The genomes FROM the donors will be added TO the recipient and the donors will be unaltered')
 
+	parser.add_argument('--donor_file',        dest = 'donor_file',    default = None,   help =  'Alternative way to supply donors. A file containing paths to the donor databases, 1 per line')
+
 	parser.add_argument('-r', '--recipient',   dest = 'recipient', default = None, help =  'Path to the recipient database. Any genomes FROM the donor database not already in the recipient will be added to this database.')	
 		
 	parser.add_argument('--verbose',           dest = 'verbose',   action='store_true', help = 'Print minor updates to console. Major updates are printed regardless.')
@@ -2720,16 +2722,23 @@ def merge_db_thread_starter(rev_index, per_db_accs):
 	
 	
 	
-def merge_db(recipient, donors, verbose, threads):
+def merge_db(recipient, donors, donor_file, verbose, threads):
 	#Prettier on the CLI
-
+	
+	if donor_file is not None:
+		fh = agnostic_reader(donor_file)
+		donors = [line.strip() for line in fh]
+		fh.close()
+	
 	if donors is None or recipient is None:
 		print("Either donor or target not given. FastAAI is exiting.")
 		return None
 	
 	print("")
 	
-	donors = donors.split(",")
+	if donor_file is None:
+		donors = donors.split(",")
+	
 	valid_donors = []
 	for d in donors:
 		if os.path.exists(d):
@@ -3454,10 +3463,11 @@ def main():
 			
 		recipient = opts.recipient
 		donors = opts.donors
+		donor_file = opts.donor_file
 		verbose = opts.verbose
 		threads = opts.threads
 		
-		merge_db(recipient, donors, verbose, threads)
+		merge_db(recipient, donors, donor_file, verbose, threads)
 		
 #################### Query files vs DB ########################	
 		

data/utils/distance/commands.rb

@@ -151,22 +151,36 @@ module MiGA::DistanceRunner::Commands
       donors << tgt_idx if tgt_idx
     end
     return nil if donors.empty?
-    run_cmd <<~CMD
-              FastAAI merge_db --donors "#{donors.join(',')}" \
-                --recipient "#{f1 = tmp_file}" --threads #{opts[:thr]}
-            CMD
+
+    # Build target database
+    f1 = tmp_file
+    if donors.size == 1
+      File.copy(donors.first, f1)
+    else
+      File.open(f0 = tmp_file, 'w') { |fh| donors.each { |i| fh.puts i } }
+      run_cmd(
+        <<~CMD
+          FastAAI merge_db --threads #{opts[:thr]} \
+            --donor_file "#{f0}" --recipient "#{f1}"
+        CMD
+      )
+      raise "Cannot merge databases into: #{f1}" unless File.size?(f1)
+    end
 
     # Run FastAAI
-    run_cmd <<~CMD
-              FastAAI db_query --query "#{qry_idx}" --target "#{f1}" \
-                --output "#{f2 = tmp_file}" --threads #{opts[:thr]} \
-                --do_stdev
-            CMD
+    run_cmd(
+      <<~CMD
+        FastAAI db_query --query "#{qry_idx}" --target "#{f1}" \
+          --output "#{f2 = tmp_file}" --threads #{opts[:thr]} \
+          --do_stdev
+      CMD
+    )
+    raise "Cannot find FastAAI output directory: #{f2}" unless Dir.exist?(f2)
 
     # Save values in the databases
     haai_data = {}
     aai_data = {}
-    # Ugly workaround to the insistence of FastAAI to not provide the files
+    # Ugly workaround to the insistence of FastAAI not to provide the files
     # I ask for ;-)
     qry_results = File.basename(qry_idx, '.faix') + '_results.txt'
     out_file = File.join(f2, 'results', qry_results)
@@ -214,6 +228,6 @@ module MiGA::DistanceRunner::Commands
 
   def run_cmd(cmd)
     puts "CMD: #{cmd}"
-    `#{cmd}`
+    puts `#{cmd} 2>&1`
   end
 end

data/utils/distance/database.rb

@@ -127,6 +127,7 @@ module MiGA::DistanceRunner::Database
     db = tmp_dbs[metric]
     table = metric == :haai ? :aai : metric
     SQLite3::Database.new(db) do |conn|
+      conn.execute('BEGIN TRANSACTION')
       data.each do |k, v|
         sql = <<~SQL
                 insert into #{table} (
@@ -135,6 +136,7 @@ module MiGA::DistanceRunner::Database
               SQL
         conn.execute(sql, [dataset.name, k] + v)
       end
+      conn.execute('COMMIT')
     end
     checkpoint(metric)
   end

data/utils/find-medoid.R

@@ -5,26 +5,28 @@
 #
 
 #= Load stuff
-argv <- commandArgs(trailingOnly = T)
+argv <- commandArgs(trailingOnly = TRUE)
 suppressPackageStartupMessages(library(ape))
-if(Sys.getenv('MIGA') == ''){
+if(Sys.getenv("MIGA") == ""){
   suppressPackageStartupMessages(library(enveomics.R))
 }else{
-  source(file.path(Sys.getenv('MIGA'),
-    'utils', 'enveomics', 'enveomics.R', 'R', 'df2dist.R'))
+  source(file.path(
+    Sys.getenv("MIGA"),
+    "utils", "enveomics", "enveomics.R", "R", "df2dist.R"
+  ))
 }
 
-find_medoids <- function(ani.df, out, clades) {
+find_medoids <- function (ani.df, out, clades) {
   if(nrow(ani.df) == 0) return(NULL)
   ani.df$d <- 1 - (ani.df$value/100)
-  dist <- enve.df2dist(ani.df, 'a', 'b', 'd', default.d = max(ani.df$d)*1.2)
+  dist <- enve.df2dist(ani.df, "a", "b", "d", default.d = max(ani.df$d) * 1.2)
   dist <- as.matrix(dist)
-  cl <- read.table(clades, header = FALSE, sep = '\t', as.is = TRUE)[,1]
+  cl <- read.table(clades, header = FALSE, sep = "\t", as.is = TRUE)[,1]
   cl.s <- c()
   medoids <- c()
   for(i in cl){
-    lab <- strsplit(i, ',')[[1]]
-    cat('Clade of:', lab[1], '\n')
+    lab <- strsplit(i, ",")[[1]]
+    cat("Clade of:", lab[1], "\n")
     if(length(lab) == 1) {
       lab.s <- lab
     } else {
@@ -32,15 +34,17 @@ find_medoids <- function(ani.df, out, clades) {
     }
     med <- lab.s[1]
     medoids <- c(medoids, med)
-    cl.s <- c(cl.s, paste(lab.s, collapse = ','))
+    cl.s <- c(cl.s, paste(lab.s, collapse = ","))
   }
   write.table(medoids, out, quote = FALSE, row.names = FALSE, col.names = FALSE)
-  write.table(cl.s, paste(clades, '.sorted', sep = ''), quote = FALSE,
-    row.names = FALSE, col.names = FALSE)
+  write.table(
+    cl.s, paste(clades, ".sorted", sep = ""), quote = FALSE,
+    row.names = FALSE, col.names = FALSE
+  )
 }
 
 #= Main
-load(argv[1])
-if(! exists('ani')) ani <- aai
+cat("Finding Medoids")
+ani <- readRDS(argv[1])
 find_medoids(ani.df = ani, out = argv[2], clades = argv[3])
 

data/utils/subclade/pipeline.rb

@@ -7,9 +7,12 @@ module MiGA::SubcladeRunner::Pipeline
       aai90: [:aai_distances, opts[:gsp_aai], :aai]
     }
     tasks.each do |k, par|
+      # Run only the requested metric
+      next unless par[2].to_s == opts[:gsp_metric]
+
       # Final output
       ogs_file = "miga-project.#{k}-clades"
-      next if File.size? ogs_file
+      next if File.size?(ogs_file)
 
       # Build ABC files
       abc_path = tmp_file("#{k}.abc")
@@ -20,7 +23,7 @@ module MiGA::SubcladeRunner::Pipeline
           next if ln =~ /^a\tb\tvalue\t/
 
           r = ln.chomp.split("\t")
-          ofh.puts "G>#{r[0]}\tG>#{r[1]}\t#{r[2]}" if r[2].to_f >= par[1]
+          ofh.puts("G>#{r[0]}\tG>#{r[1]}\t#{r[2]}") if r[2].to_f >= par[1]
         end
       end
       ofh.close
@@ -29,22 +32,20 @@ module MiGA::SubcladeRunner::Pipeline
         `ogs.mcl.rb -o '#{ogs_file}.tmp' --abc '#{abc_path}' -t '#{opts[:thr]}'`
         File.open(ogs_file, 'w') do |fh|
           File.foreach("#{ogs_file}.tmp").with_index do |ln, lno|
-            fh.puts ln if lno > 0
+            fh.puts(ln) if lno > 0
           end
         end
         File.unlink "#{ogs_file}.tmp"
       else
-        FileUtils.touch ogs_file
-      end
-      if par[2].to_s == opts[:gsp_metric]
-        FileUtils.cp(ogs_file, "miga-project.gsp-clades")
+        FileUtils.touch(ogs_file)
       end
+      FileUtils.cp(ogs_file, 'miga-project.gsp-clades')
     end
 
     # Find genomospecies medoids
     src = File.expand_path('utils/find-medoid.R', MiGA::MiGA.root_path)
     dir = opts[:gsp_metric] == 'aai' ? '02.aai' : '03.ani'
-    `Rscript '#{src}' ../../09.distances/#{dir}/miga-project.Rdata \
+    `Rscript '#{src}' '../../09.distances/#{dir}/miga-project.rds' \
       miga-project.gsp-medoids miga-project.gsp-clades`
     if File.exist? 'miga-project.gsp-clades.sorted'
       File.rename 'miga-project.gsp-clades.sorted', 'miga-project.gsp-clades'
@@ -54,8 +55,6 @@ module MiGA::SubcladeRunner::Pipeline
     ofh = File.open('miga-project.proposed-clades', 'w')
     File.open('miga-project.gsp-clades', 'r') do |ifh|
       ifh.each_line do |ln|
-        next if $. == 1
-
         r = ln.chomp.split(',')
         ofh.puts r.join("\t") if r.size >= 5
       end
@@ -69,7 +68,7 @@ module MiGA::SubcladeRunner::Pipeline
     metric_res = project.result(step) or raise "Incomplete step #{step}"
     matrix = metric_res.file_path(:matrix)
     `Rscript '#{src}' '#{matrix}' miga-project '#{opts[:thr]}' \
-      miga-project.ani95-medoids '#{opts[:run_clades] ? 'cluster' : 'empty'}'`
+      miga-project.gsp-medoids '#{opts[:run_clades] ? 'cluster' : 'empty'}'`
     if File.exist? 'miga-project.nwk'
       File.rename('miga-project.nwk', "miga-project.#{metric}.nwk")
     end

data/utils/subclades.R

@@ -10,56 +10,55 @@ suppressPackageStartupMessages(library(ape))
 suppressPackageStartupMessages(library(vegan))
 suppressPackageStartupMessages(library(cluster))
 suppressPackageStartupMessages(library(parallel))
-if(Sys.getenv('MIGA') == ''){
+if(Sys.getenv("MIGA") == ""){
   suppressPackageStartupMessages(library(enveomics.R))
 }else{
-  source(file.path(Sys.getenv('MIGA'),
-    'utils', 'enveomics', 'enveomics.R', 'R', 'df2dist.R'))
+  source(file.path(
+    Sys.getenv("MIGA"),
+    "utils", "enveomics", "enveomics.R", "R", "df2dist.R"
+  ))
 }
 
 #= Main function
 subclades <- function(ani_file, out_base, thr = 1, ani.d = dist(0), sel = NA) {
-  say('==> Out base:', out_base, '<==')
+  say("==> Out base:", out_base, "<==")
 
   # Normalize input matrix
-  dist_rdata = paste(out_base, 'dist.rdata', sep = '.')
-  if(!missing(ani_file)){
-    if(length(ani.d) == 0 && !file.exists(dist_rdata)){
-      # Read from ani_file
-      a <- read.table(gzfile(ani_file), sep = '\t', header = TRUE, as.is = TRUE)
-      if(nrow(a) == 0){
-        generate_empty_files(out_base)
-        return(NULL)
-      }
-      if(!is.na(sel) && file.exists(sel)){
-        say('Filter selection')
-        lab <- read.table(sel, sep='\t', head=FALSE, as.is=TRUE)[,1]
-        a <- a[a$a %in% lab & a$b %in% lab, ]
+  dist_rds <- paste(out_base, "dist.rds", sep = ".")
+  if (!missing(ani_file)) {
+    if (length(ani.d) == 0) {
+      if (file.exists(dist_rds)) {
+        ani.d <- readRDS(dist_rds)
+      } else {
+        # Read from ani_file
+        ani.d <- ani_distance(ani_file, sel)
+        if (is.null(ani.d)) {
+          generate_empty_files(out_base)
+          return(NULL)
+        } else {
+          saveRDS(ani.d, dist_rds)
+        }
       }
-      say('Distances')
-      a$d <- 1 - (a$value/100)
-      ani.d <- enve.df2dist(a, 'a', 'b', 'd', default.d = max(a$d)*1.2)
-      save(ani.d, file = dist_rdata)
     }
   }
 
   # Read result if the subclade is ready, run it otherwise
-  if(file.exists(paste(out_base, 'classif', sep = '.'))){
+  if (file.exists(paste(out_base, "classif", sep = "."))) {
     say("Loading")
     ani.medoids <- read.table(paste(out_base, "medoids", sep = "."),
-      sep = ' ', as.is = TRUE)[,1]
+      sep = " ", as.is = TRUE)[,1]
     a <- read.table(paste(out_base, "classif", sep="."),
-      sep = '\t', as.is = TRUE)
+      sep = "\t", as.is = TRUE)
     ani.types <- a[,2]
     names(ani.types) <- a[,1]
-    if(length(ani.d) == 0) load(dist_rdata)
-  }else if(length(labels(ani.d)) > 8L){
-    res <- subclade_clustering(out_base, thr, ani.d, dist_rdata)
-    if(length(res) == 0) return(NULL)
-    ani.medoids <- res[['ani.medoids']]
-    ani.types <- res[['ani.types']]
-    ani.d <- res[['ani.d']]
-  }else{
+    if(length(ani.d) == 0) ani.d <- readRDS(dist_rds)
+  } else if (length(labels(ani.d)) > 8L) {
+    res <- subclade_clustering(out_base, thr, ani.d, dist_rds)
+    if (length(res) == 0) return(NULL)
+    ani.medoids <- res[["ani.medoids"]]
+    ani.types <- res[["ani.types"]]
+    ani.d <- res[["ani.d"]]
+  } else {
     ani.medoids <- labels(ani.d)[which.min(colSums(as.matrix(ani.d)))]
     ani.types <- rep(1, length(labels(ani.d)))
     names(ani.types) <- labels(ani.d)
@@ -69,66 +68,69 @@ subclades <- function(ani_file, out_base, thr = 1, ani.d = dist(0), sel = NA) {
 
   # Recursive search
   say("Recursive search")
-  for(i in 1:length(ani.medoids)){
+  for (i in 1:length(ani.medoids)) {
     medoid <- ani.medoids[i]
     ds_f <- names(ani.types)[ ani.types==i ]
     say("Analyzing subclade", i, "with medoid:", medoid)
     dir_f <- paste(out_base, ".sc-", i, sep="")
-    if(!dir.exists(dir_f)) dir.create(dir_f)
+    if (!dir.exists(dir_f)) dir.create(dir_f)
     write.table(ds_f,
       paste(out_base, ".sc-", i, "/miga-project.all",sep=""),
       quote=FALSE, col.names=FALSE, row.names=FALSE)
-    if(length(ds_f) > 8L){
+    if (length(ds_f) > 8L) {
       ani_subset <- as.dist(as.matrix(ani.d)[ds_f, ds_f])
-      subclades(out_base=paste(out_base, ".sc-", i, "/miga-project", sep=""),
-        thr=thr, ani.d=ani_subset)
+      subclades(
+        out_base = paste(out_base, ".sc-", i, "/miga-project", sep=""),
+        thr = thr,
+        ani.d = ani_subset
+      )
     }
   }
   
   # Declare recursion up-to-here complete
-  write.table(date(), paste(out_base, 'ready', sep='.'),
-        quote=FALSE, row.names=FALSE, col.names=FALSE)
+  write.table(
+    date(), paste(out_base, "ready", sep = "."),
+    quote = FALSE, row.names = FALSE, col.names = FALSE
+  )
 }
 
 #= Heavy-lifter
-subclade_clustering <- function(out_base, thr, ani.d, dist_rdata) {
+subclade_clustering <- function (out_base, thr, ani.d, dist_rds) {
   # Get ANI distances
-  if(length(ani.d) > 0){
-    # Just use ani.d (and save in dist_rdata_
-    save(ani.d, file=dist_rdata)
-  }else if(file.exists(dist_rdata)){
-    # Read from dist_rdata
-    load(dist_rdata)
-  }else{
+  if (length(ani.d) > 0) {
+    # Just use ani.d (and save in dist_rds)
+    if (!file.exists(dist_rds)) saveRDS(ani.d, dist_rds)
+  } else if (file.exists(dist_rds)) {
+    # Read from dist_rds
+    ani.d <- readRDS(dist_rds)
+  } else {
     stop("Cannot find input matrix", out_base)
   }
-  if(length(labels(ani.d)) <= 8L) return(list())
+  if (length(labels(ani.d)) <= 8L) return(list())
   
-  # Build tree
-  say("Tree")
-  ani.ph <- bionj(ani.d)
-  express.ori <- options('expressions')$expressions
-  if(express.ori < ani.ph$Nnode*4){
-    options(expressions=min(c(5e7,ani.ph$Nnode*4)))
-  }
-  write.tree(ani.ph, paste(out_base, ".nwk", sep=""))
-  options(expressions=express.ori)
-
   # Silhouette
   say("Silhouette")
   nn <- length(labels(ani.d))
   k <- min(max(floor(0.005 * nn), 2), 20):min(nn-1, 100)
+  say("- Make cluster")
   cl <- makeCluster(thr)
-  s <- parSapply(cl, k, function(x) {
+  say("- Launch parallel jobs")
+  s <- parSapply(
+    cl, k,
+    function(x) {
       library(cluster)
-      s <- pam(ani.d, x, do.swap=FALSE, pamonce=1)$silinfo
-      c(s$avg.width, -sum(ifelse(s$widths[,3]>0,0,s$widths[,3])))
-    })
+      s <- pam(ani.d, x, do.swap = FALSE, pamonce = 1)$silinfo
+      c(s$avg.width, -sum(ifelse(s$widths[,3] > 0, 0, s$widths[,3])))
+    }
+  )
+  say("- Stop cluster")
   stopCluster(cl)
-  s.avg.z <- (s[1,]-mean(s[1,]))/(sd(s[1,])+0.0001)
-  s.neg.z <- (s[2,]-mean(s[2,]))/(sd(s[2,])+0.01)
-  ds <- s.avg.z - s.neg.z - 2/(1:length(k)) - (1:length(k))/50
-  if(mean(s[1,]<0)<0.75) ds[s[1,]<0] <- mean(ds) # <- k's with negative average
+  say("- Calculate custom criteria")
+  s.avg.z <- (s[1,] - mean(s[1,])) / (sd(s[1,]) + 0.0001)
+  s.neg.z <- (s[2,] - mean(s[2,])) / (sd(s[2,]) + 0.01)
+  ds <- s.avg.z - s.neg.z - 2 / (1:length(k)) - (1:length(k)) / 50
+  if(mean(s[1,] < 0) < 0.75)
+    ds[s[1,] < 0] <- mean(ds) # <- k's with negative average
   top.n <- k[which.max(ds)]
 
   # Classify genomes
@@ -137,10 +139,21 @@ subclade_clustering <- function(out_base, thr, ani.d, dist_rdata) {
   ani.types <- ani.cl$clustering
   ani.medoids <- ani.cl$medoids
 
+  # Build tree
+  say("Tree")
+  ani.ph <- bionj(ani.d)
+  say("- Write")
+  express.ori <- options("expressions")$expressions
+  if(express.ori < ani.ph$Nnode * 4){
+    options(expressions=min(c(5e7, ani.ph$Nnode * 4)))
+  }
+  write.tree(ani.ph, paste(out_base, ".nwk", sep = ""))
+  options(expressions=express.ori)
+
   # Generate graphic report
   say("Graphic report")
-  pdf(paste(out_base, ".pdf", sep=""), 7, 12)
-  layout(matrix(c(1,1,2,2,3,3,4,5),byrow=TRUE, ncol=2))
+  pdf(paste(out_base, ".pdf", sep = ""), 7, 12)
+  layout(matrix(c(rep(1:3, each = 2), 4:5), byrow = TRUE, ncol = 2))
   plot_distances(ani.d)
   plot_silhouette(k, s[1,], s[2,], ds, top.n)
   plot_clustering(ani.cl, ani.d, ani.types)
@@ -153,112 +166,170 @@ subclade_clustering <- function(out_base, thr, ani.d, dist_rdata) {
   # Return data
   say("Cluster ready")
   return(list(
-    ani.medoids=ani.medoids,
-    ani.types=ani.types,
-    ani.d=ani.d
+    ani.medoids = ani.medoids,
+    ani.types = ani.types,
+    ani.d = ani.d
   ))
 }
 
 #= Helper functions
-say <- function(...) { message(paste("[",date(),"]",...,"\n"),appendLF=FALSE) }
+say <- function (...) {
+  message(paste("[", date(), "]", ..., "\n"), appendLF = FALSE)
+}
 
-generate_empty_files <- function(out_base) {
-  pdf(paste(out_base, ".pdf", sep=""), 7, 12)
-  plot(1, t="n", axes=F)
-  legend("center", "No data", bty="n")
+generate_empty_files <- function (out_base) {
+  pdf(paste(out_base, ".pdf", sep = ""), 7, 12)
+  plot(1, t = "n", axes = F)
+  legend("center", "No data", bty = "n")
   dev.off()
-  file.create(paste(out_base,".1.classif",sep=""))
-  file.create(paste(out_base,".1.medoids",sep=""))
+  file.create(paste(out_base, ".1.classif", sep = ""))
+  file.create(paste(out_base, ".1.medoids", sep = ""))
 }
 
-write_text_report <- function(out_base, ani.d, ani.medoids, ani.types){
-  say('Text report')
-  write.table(ani.medoids, paste(out_base, 'medoids', sep='.'),
-    quote=FALSE, col.names=FALSE, row.names=FALSE)
-  classif <- cbind(names(ani.types), ani.types, ani.medoids[ ani.types ], NA)
-  ani.d.m <- 100 - as.matrix(ani.d)*100
-  for(j in 1:nrow(classif)){
+write_text_report <- function (out_base, ani.d, ani.medoids, ani.types) {
+  say("Text report")
+  write.table(
+    ani.medoids, paste(out_base, "medoids", sep = "."),
+    quote = FALSE, col.names = FALSE, row.names = FALSE
+  )
+  classif <- cbind(names(ani.types), ani.types, ani.medoids[ani.types], NA)
+  ani.d.m <- 100 - as.matrix(ani.d) * 100
+  for (j in 1:nrow(classif)) {
     classif[j,4] <- ani.d.m[classif[j,1], classif[j,3]]
   }
-  write.table(classif, paste(out_base,"classif",sep="."),
-    quote=FALSE, col.names=FALSE, row.names=FALSE, sep="\t")
+  write.table(
+    classif, paste(out_base, "classif", sep="."),
+    quote = FALSE, col.names = FALSE, row.names = FALSE, sep = "\t"
+  )
 }
 
-plot_silhouette <- function(k, s, ns, ds, top.n) {
+plot_silhouette <- function (k, s, ns, ds, top.n) {
   # s
-  par(mar=c(4,5,1,5)+0.1)
-  plot(1, t="n", xlab="k (clusters)", ylab="", xlim=range(c(0,k)),
-    ylim=range(s), bty="n", xaxs="i", yaxt="n")
-  polygon(c(k[1], k, k[length(k)]), c(0,s,0), border=NA, col="grey80")
-  axis(2, fg="grey60", col.axis="grey60")
-  mtext("Mean silhouette", side=2, line=3, col="grey60")
+  par(mar = c(4,5,1,5)+0.1)
+  plot(
+    1, t = "n", xlab = "k (clusters)", ylab = "", xlim = range(c(0,k)),
+    ylim = range(s), bty = "n", xaxs = "i", yaxt = "n"
+  )
+  polygon(c(k[1], k, k[length(k)]), c(0,s,0), border = NA, col = "grey80")
+  axis(2, fg = "grey60", col.axis = "grey60")
+  mtext("Mean silhouette", side = 2, line = 3, col = "grey60")
+
   # ns
-  par(new=TRUE)
-  plot(1, t="n", xlab="", xaxt="n", ylab="", yaxt="n", xlim=range(c(0,k)),
-    ylim=range(ns), bty="n", xaxs="i")
-  points(k, ns, type="o", pch=16, col=rgb(1/2,0,0,3/4))
-  axis(4, fg="darkred", col.axis="darkred")
-  mtext("Negative silhouette area", side=4, line=3, col="darkred")
+  par(new = TRUE)
+  plot(
+    1, t = "n", bty = "n",
+    xlab = "", ylab = "", xaxt = "n", yaxt = "n", xaxs = "i",
+    xlim = range(c(0,k)), ylim = range(ns)
+  )
+  points(k, ns, type = "o", pch = 16, col = rgb(1/2,0,0,3/4))
+  axis(4, fg = "darkred", col.axis = "darkred")
+  mtext("Negative silhouette area", side = 4, line = 3, col = "darkred")
+
   # ds
-  par(new=TRUE)
-  plot(1, t="n", xlab="", xaxt="n", ylab="", yaxt="n", xlim=range(c(0,k)),
-    ylim=range(ds), bty="n", xaxs="i")
+  par(new = TRUE)
+  plot(
+    1, t = "n", bty = "n",
+    xlab = "", ylab = "", xaxt = "n", yaxt = "n", xaxs = "i",
+    xlim = range(c(0,k)), ylim = range(ds)
+  )
   lines(k, ds)
-  abline(v=top.n, lty=2)
+  abline(v = top.n, lty = 2)
 }
 
-plot_distances <- function(dist) {
-  par(mar=c(5,4,1,2)+0.1)
-  hist(dist, border=NA, col="grey60", breaks=50, xlab="Distances", main="")
+plot_distances <- function (dist) {
+  par(mar = c(5,4,1,2) + 0.1)
+  hist(
+    dist, border = NA, col = "grey60", breaks = 50,
+    xlab = "Distances", main = ""
+  )
 }
 
-plot_clustering <- function(cl, dist, types) {
-  par(mar=c(5,4,4,2)+0.1)
+plot_clustering <- function (cl, dist, types) {
+  par(mar = c(5,4,4,2) + 0.1)
   top.n <- length(cl$medoids)
   col <- ggplotColours(top.n)
-  plot(silhouette(cl), col=col)
-  if(length(labels(dist))<=15){
-    plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
-    plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
-  }else{
-    ani.mds <- cmdscale(dist, k=4)
-    if(ncol(ani.mds)==4){
-      plot(ani.mds[,1], ani.mds[,2], col=col[types], cex=1/2,
-	xlab='Component 1', ylab='Component 2')
-      plot(ani.mds[,3], ani.mds[,4], col=col[types], cex=1/2,
-	xlab='Component 3', ylab='Component 4')
+  plot(silhouette(cl), col = col)
+  if (length(labels(dist)) <= 15) {
+    plot(1, type = "n", axes = FALSE, xlab = "", ylab = "", bty = "n")
+    plot(1, type = "n", axes = FALSE, xlab = "", ylab = "", bty = "n")
+  } else {
+    ani.mds <- cmdscale(dist, k = 4)
+    if (ncol(ani.mds) == 4) {
+      plot(
+        ani.mds[,1], ani.mds[,2], col = col[types], cex = 1/2,
+	xlab = "Component 1", ylab = "Component 2"
+      )
+      plot(
+        ani.mds[,3], ani.mds[,4], col = col[types], cex = 1/2,
+	xlab = "Component 3", ylab="Component 4"
+      )
     }else{
-      plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
-      plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
+      for (i in 1:2)
+        plot(1, type = "n", axes = FALSE, xlab = "", ylab = "", bty = "n")
     }
   }
 }
 
-plot_tree <- function(phy, types, medoids){
+plot_tree <- function (phy, types, medoids) {
   layout(1)
   top.n <- length(unique(types))
   col <- ggplotColours(top.n)
   is.medoid <- phy$tip.label %in% medoids
-  phy$tip.label[is.medoid] <- paste(phy$tip.label[is.medoid],
-    " [", types[phy$tip.label[is.medoid]], "]", sep='')
-  plot(phy, cex=ifelse(is.medoid, 1/3, 1/6),
-    font=ifelse(is.medoid, 2, 1),
-    tip.color=col[types[phy$tip.label]])
+  phy$tip.label[is.medoid] <- paste(
+    phy$tip.label[is.medoid],
+    " [", types[phy$tip.label[is.medoid]], "]",
+    sep = ""
+  )
+  plot(
+    phy, cex = ifelse(is.medoid, 1/3, 1/6),
+    font = ifelse(is.medoid, 2, 1),
+    tip.color = col[types[phy$tip.label]]
+  )
 }
 
-ggplotColours <- function(n=6, h=c(0, 360)+15, alpha=1){
-  if ((diff(h)%%360) < 1) h[2] <- h[2] - 360/n
-  hcl(h=seq(h[1], h[2], length=n), c=100, l=65, alpha=alpha)
+ggplotColours <- function (n = 6, h = c(0, 360) + 15, alpha = 1) {
+  if ((diff(h) %% 360) < 1) h[2] <- h[2] - 360 / n
+  hcl(h = seq(h[1], h[2], length = n), c = 100, l = 65, alpha = alpha)
+}
+
+ani_distance <- function (ani_file, sel) {
+  # Try to locate rds, otherwise read gzipped table
+  rds <- gsub("\\.txt\\.gz$", ".rds", ani_file)
+  if (file.exists(rds)) {
+    sim <- readRDS(rds)
+  } else {
+    sim <- read.table(gzfile(ani_file), sep = "\t", header = TRUE, as.is = TRUE)
+  }
+  
+  # If there is not data end process
+  if (nrow(sim) == 0) return(NULL)
+
+  # Apply filter (if requested)
+  if (!is.na(sel) && file.exists(sel)) {
+    say("Filter selection")
+    lab <- read.table(sel, sep = "\t", head = FALSE, as.is = TRUE)[,1]
+    sim <- sim[sim$a %in% lab & sim$b %in% lab, ]
+  }
+
+  # Transform to distances
+  say("Distances")
+  sim$d <- 1 - (sim$value / 100)
+  return(enve.df2dist(sim, "a", "b", "d", default.d = max(sim$d) * 1.2))
 }
 
 #= Main
 options(warn = 1)
-if(length(argv) >= 5 & argv[5] == 'empty'){
+if (length(argv) >= 5 & argv[5] == "empty") {
   generate_empty_files(argv[2])
-  write.table(NULL, paste(argv[2], "medoids", sep="."))
-  write.table(NULL, paste(argv[2], "classif", sep="."))
-  write.table(date(), paste(argv[2], "ready", sep="."))
+  write.table(NULL, paste(argv[2], "medoids", sep = "."))
+  write.table(NULL, paste(argv[2], "classif", sep = "."))
+  write.table(date(), paste(argv[2], "ready", sep = "."))
 }else{
-  subclades(ani_file = argv[1], out_base = argv[2],
-    thr = ifelse(is.na(argv[3]), 1, as.numeric(argv[3])), sel = argv[4])
+  subclades(
+    ani_file = argv[1],
+    out_base = argv[2],
+    thr = ifelse(is.na(argv[3]), 1, as.numeric(argv[3])),
+    sel = argv[4]
+  )
 }
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.1.2.4
+  version: 1.1.3.3
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-16 00:00:00.000000000 Z
+date: 2021-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons