miga-base 0.7.4.0 → 0.7.9.0

data/utils/distance/commands.rb

@@ -1,12 +1,13 @@
-
 module MiGA::DistanceRunner::Commands
   # Estimates or calculates AAI against +target+
   def aai(target)
     # Check if the request makes sense
     return nil if target.nil? || target.result(:essential_genes).nil?
+
     # Check if it's been calculated
     y = stored_value(target, :aai)
     return y unless y.nil? || y.zero?
+
     # Try hAAI (except in clade projects)
     unless @ref_project.is_clade?
       y = haai(target)
@@ -14,24 +15,27 @@ module MiGA::DistanceRunner::Commands
     end
     # Full AAI
     aai_cmd(
-          tmp_file('proteins.fa'), target.result(:cds).file_path(:proteins),
-          dataset.name, target.name, tmp_dbs[:aai]).tap{ checkpoint :aai }
+      tmp_file('proteins.fa'), target.result(:cds).file_path(:proteins),
+      dataset.name, target.name, tmp_dbs[:aai]
+    ).tap { checkpoint :aai }
   end
 
   ##
   # Estimates AAI against +target+ using hAAI
   def haai(target)
     return nil if opts[:haai_p] == 'no'
+
     haai = aai_cmd(tmp_file('ess_genes.fa'),
-          target.result(:essential_genes).file_path(:ess_genes),
-          dataset.name, target.name, tmp_dbs[:haai],
-          aai_save_rbm: 'no-save-rbm', aai_p: opts[:haai_p])
+                   target.result(:essential_genes).file_path(:ess_genes),
+                   dataset.name, target.name, tmp_dbs[:haai],
+                   aai_save_rbm: 'no-save-rbm', aai_p: opts[:haai_p])
     checkpoint :haai
     return nil if haai.nil? || haai.zero? || haai > 90.0
-    aai = 100.0 - Math.exp(2.435076 + 0.4275193*Math.log(100.0-haai))
+
+    aai = 100.0 - Math.exp(2.435076 + 0.4275193 * Math.log(100.0 - haai))
     SQLite3::Database.new(tmp_dbs[:aai]) do |conn|
       conn.execute 'insert into aai values(?, ?, ?, 0, 0, 0)',
-            [dataset.name, target.name, aai]
+                   [dataset.name, target.name, aai]
     end
     checkpoint :aai
     aai
@@ -44,13 +48,16 @@ module MiGA::DistanceRunner::Commands
     t = tmp_file('largecontigs.fa')
     r = target.result(:assembly)
     return nil if r.nil? || !File.size?(t)
+
     # Check if it's been calculated
     y = stored_value(target, :ani)
     return y unless y.nil? || y.zero?
+
     # Run it
     ani_cmd(
-        t, r.file_path(:largecontigs),
-        dataset.name, target.name, tmp_dbs[:ani]).tap{ checkpoint :ani }
+      t, r.file_path(:largecontigs),
+      dataset.name, target.name, tmp_dbs[:ani]
+    ).tap { checkpoint :ani }
   end
 
   ##
@@ -74,7 +81,7 @@ module MiGA::DistanceRunner::Commands
 
   ##
   # Execute an ANI command
-  def ani_cmd(f1, f2, n1, n2, db, o={})
+  def ani_cmd(f1, f2, n1, n2, db, o = {})
     o = opts.merge(o)
     v = nil
     if o[:ani_p] == 'fastani'
@@ -83,7 +90,7 @@ module MiGA::DistanceRunner::Commands
       unless out.empty?
         SQLite3::Database.new(db) do |conn|
           conn.execute 'insert into ani values(?, ?, ?, 0, ?, ?)',
-                [n1, n2, out[2], out[3], out[4]]
+                       [n1, n2, out[2], out[3], out[4]]
         end
       end
       v = out[2]

data/utils/distance/database.rb

@@ -1,4 +1,3 @@
-
 require 'sqlite3'
 
 module MiGA::DistanceRunner::Database
@@ -9,7 +8,7 @@ module MiGA::DistanceRunner::Database
     @dbs = {}
     @tmp_dbs = {}
     @db_counts = {}
-    {haai: :aai, aai: :aai, ani: :ani}.each do |m, t|
+    { haai: :aai, aai: :aai, ani: :ani }.each do |m, t|
       @db_counts[m] = 0
       @dbs[m] = for_ref ? ref_db(m) : query_db(m)
       # Remove if corrupt
@@ -25,9 +24,9 @@ module MiGA::DistanceRunner::Database
       # Initialize if it doesn't exist
       SQLite3::Database.new(dbs[m]) do |conn|
         conn.execute "create table if not exists #{t}(" +
-            "seq1 varchar(256), seq2 varchar(256), " +
-            "#{t} float, sd float, n int, omega int" +
-          ")"
+                     "seq1 varchar(256), seq2 varchar(256), " +
+                     "#{t} float, sd float, n int, omega int" +
+                     ")"
       end unless File.size? dbs[m]
       # Copy over to (local) temporals
       @tmp_dbs[m] = tmp_file("#{m}.db")
@@ -38,16 +37,17 @@ module MiGA::DistanceRunner::Database
   ##
   # Path to the database +metric+ for +dataset_name+ in +project+
   # (assumes that +dataset_name+ is a reference dataset)
-  def ref_db(metric, dataset_name=nil)
+  def ref_db(metric, dataset_name = nil)
     dataset_name ||= dataset.name
-    b = case metric
-    when :haai
-      "01.haai/#{dataset_name}.db"
-    when :aai
-      "02.aai/#{dataset_name}.db"
-    when :ani
-      "03.ani/#{dataset_name}.db"
-    end
+    b =
+      case metric
+      when :haai
+        "01.haai/#{dataset_name}.db"
+      when :aai
+        "02.aai/#{dataset_name}.db"
+      when :ani
+        "03.ani/#{dataset_name}.db"
+      end
     File.expand_path(b, home)
   end
 
@@ -63,13 +63,14 @@ module MiGA::DistanceRunner::Database
   def stored_value(target, metric)
     # Check if self.dataset -> target is done (previous run)
     y = value_from_db(dataset.name, target.name, tmp_dbs[metric], metric)
-    return y unless y.nil? or y.zero?
+    return y unless y.nil? || y.zero?
 
     # Check if self.dataset <- target is done (another thread)
-    if dataset.is_ref? and project.path == ref_project.path
+    if dataset.is_ref? && project.path == ref_project.path
       y = data_from_db(
-        target.name, dataset.name, ref_db(metric, target.name), metric)
-      unless y.nil? or y.first.nil? or y.first.zero?
+        target.name, dataset.name, ref_db(metric, target.name), metric
+      )
+      unless y.nil? || y.first.nil? || y.first.zero?
         # Store a copy
         data_to_db(dataset.name, target.name, tmp_dbs[metric], metric, y)
         return y.first
@@ -94,7 +95,8 @@ module MiGA::DistanceRunner::Database
     SQLite3::Database.new(db) do |conn|
       y = conn.execute(
         "select #{metric}, sd, n, omega from #{metric} where seq1=? and seq2=?",
-        [n1, n2]).first
+        [n1, n2]
+      ).first
     end if File.size? db
     y
   end
@@ -105,7 +107,8 @@ module MiGA::DistanceRunner::Database
     SQLite3::Database.new(db) do |conn|
       conn.execute(
         "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-        "values (?, ?, ?, ?, ?, ?)", [n1, n2] + data)
+        "values (?, ?, ?, ?, ?, ?)", [n1, n2] + data
+      )
     end
     checkpoint metric
   end
@@ -114,7 +117,7 @@ module MiGA::DistanceRunner::Database
   # Iterates for each entry in +db+
   def foreach_in_db(db, metric, &blk)
     SQLite3::Database.new(db) do |conn|
-      conn.execute("select * from #{metric}").each{ |r| blk[r] }
+      conn.execute("select * from #{metric}").each { |r| blk[r] }
     end
   end
 end

data/utils/distance/pipeline.rb

@@ -1,13 +1,12 @@
-
 # High-end pipelines for DistanceRunner
 module MiGA::DistanceRunner::Pipeline
-
   # Recursively classify the dataset, returning an Array with two entries:
   # classification and cluster number
   def classify(clades, classif, metric, result_fh, val_cls = nil)
     dir = File.expand_path(classif, clades)
     med = File.expand_path('miga-project.medoids', dir)
-    return [classif,val_cls] unless File.size? med
+    return [classif, val_cls] unless File.size? med
+
     max_val = 0
     val_med = ''
     val_cls = nil
@@ -30,11 +29,22 @@ module MiGA::DistanceRunner::Pipeline
     classify(clades, classif, metric, result_fh, val_cls)
   end
 
+  # Run distances against datasets listed in metadata's +:dist_req+
+  def distances_by_request(metric)
+    return unless dataset.metadata[:dist_req]
+
+    $stderr.puts 'Running distances by request'
+    dataset.metadata[:dist_req].each do |target|
+      ds = ref_project.dataset(target) and send(metric, ds)
+    end
+  end
+
   # Builds a tree with all visited medoids from any classification level
   def build_medoids_tree(metric)
     $stderr.puts "Building medoids tree (metric = #{metric})"
     db = query_db(metric)
     return unless File.size? db
+
     out_base = File.expand_path(dataset.name, home)
     ds_matrix = "#{out_base}.txt"
     ds_matrix_fh = File.open(ds_matrix, 'w')
@@ -43,7 +53,7 @@ module MiGA::DistanceRunner::Pipeline
     seq2 = []
     foreach_in_db(db, metric) do |r|
       seq2 << r[0]
-      ds_matrix_fh.puts r[0,3].join("\t")
+      ds_matrix_fh.puts r[0, 3].join("\t")
     end
     # Find all values among visited datasets in ref_project
     ref_r = ref_project.result("#{metric}_distances") or return
@@ -51,7 +61,8 @@ module MiGA::DistanceRunner::Pipeline
       fh.each_line do |ln|
         r = ln.chomp.split("\t")
         next unless seq2.include?(r[1]) or seq2.include?(r[2])
-        ds_matrix_fh.puts r[1,3].join("\t")
+
+        ds_matrix_fh.puts r[1, 3].join("\t")
       end
     end
     ds_matrix_fh.close
@@ -74,11 +85,12 @@ module MiGA::DistanceRunner::Pipeline
     dataset.add_result(from_ref_project ? :taxonomy : :distances, true)
     cr = dataset.closest_relatives(1, from_ref_project)
     return if cr.nil? or cr.empty?
+
     tax = ref_project.dataset(cr[0][0]).metadata[:tax] || {}
 
     # Run the test for each rank
     tax_test = MiGA::TaxDist.aai_pvalues(cr[0][1], :intax, engine: opts[:aai_p])
-    r = tax_test.map do |k,v|
+    r = tax_test.map do |k, v|
       sig = ''
       [0.5, 0.1, 0.05, 0.01].each { |i| sig << '*' if v < i }
       [MiGA::Taxonomy.LONG_RANKS[k], (tax[k] || '?'), v, sig]
@@ -97,12 +109,13 @@ module MiGA::DistanceRunner::Pipeline
 
   # Transfer the taxonomy to the current dataset
   def transfer_taxonomy(tax)
-    $stderr.puts "Transferring taxonomy"
+    $stderr.puts 'Transferring taxonomy'
     return if tax.nil?
+
     pval = (project.metadata[:tax_pvalue] || 0.05).to_f
-    tax_a = tax.
-      select { |i| i[1] != '?' && i[2] <= pval }.
-      map { |i| i[0,2].join(':') }
+    tax_a = tax
+            .select { |i| i[1] != '?' && i[2] <= pval }
+            .map { |i| i[0, 2].join(':') }
     dataset.metadata[:tax] = MiGA::Taxonomy.new(tax_a)
     dataset.save
   end

data/utils/distance/runner.rb

@@ -1,13 +1,10 @@
-
 require_relative 'base.rb'
 require_relative 'temporal.rb'
 require_relative 'database.rb'
 require_relative 'commands.rb'
 require_relative 'pipeline.rb'
 
-
 class MiGA::DistanceRunner
-
   include MiGA::DistanceRunner::Temporal
   include MiGA::DistanceRunner::Database
   include MiGA::DistanceRunner::Commands
@@ -16,7 +13,7 @@ class MiGA::DistanceRunner
   attr_reader :project, :ref_project, :dataset, :opts, :home
   attr_reader :tmp, :tmp_dbs, :dbs, :db_counts
 
-  def initialize(project_path, dataset_name, opts_hash={})
+  def initialize(project_path, dataset_name, opts_hash = {})
     @opts = opts_hash
     @project = MiGA::Project.load(project_path) or
       raise "No project at #{project_path}"
@@ -30,7 +27,7 @@ class MiGA::DistanceRunner
     @opts[:aai_save_rbm] ||= ENV.fetch('MIGA_AAI_SAVE_RBM') do
       project.is_clade? ? 'save-rbm' : 'no-save-rbm'
     end
-    @opts[:thr] ||= ENV.fetch('CORES'){ 2 }.to_i
+    @opts[:thr] ||= ENV.fetch('CORES') { 2 }.to_i
     if opts[:run_taxonomy] and project.metadata[:ref_project]
       ref_path = project.metadata[:ref_project]
       @home = File.expand_path('05.taxonomy', @home)
@@ -60,6 +57,7 @@ class MiGA::DistanceRunner
   def go!
     $stderr.puts "Launching analysis"
     return if dataset.is_multi?
+
     Dir.mktmpdir do |tmp_dir|
       @tmp = tmp_dir
       create_temporals
@@ -69,25 +67,26 @@ class MiGA::DistanceRunner
 
   # Launch analysis for reference datasets
   def go_ref!
-    $stderr.puts "Launching analysis for reference dataset"
+    $stderr.puts 'Launching analysis for reference dataset'
     # Initialize databases
     initialize_dbs! true
 
     # first-come-first-serve traverse
     ref_project.each_dataset do |ds|
       next if !ds.is_ref? or ds.is_multi? or ds.result(:essential_genes).nil?
+
       puts "[ #{Time.now} ] #{ds.name}"
       ani_after_aai(ds)
     end
 
     # Finalize
-    [:haai, :aai, :ani].each{ |m| checkpoint! m if db_counts[m] > 0 }
+    %i[haai aai ani].each { |m| checkpoint! m if db_counts[m] > 0 }
   end
 
   ##
   # Launch analysis for query datasets
   def go_query!
-    $stderr.puts "Launching analysis for query dataset"
+    $stderr.puts 'Launching analysis for query dataset'
     # Check if project is ready
     tsk = ref_project.is_clade? ? [:subclades, :ani] : [:clade_finding, :aai]
     res = ref_project.result(tsk[0])
@@ -95,6 +94,7 @@ class MiGA::DistanceRunner
 
     # Initialize the databases
     initialize_dbs! false
+    distances_by_request(tsk[1])
     # Calculate the classification-informed AAI/ANI traverse
     results = File.expand_path("#{dataset.name}.#{tsk[1]}-medoids.tsv", home)
     fh = File.open(results, 'w')
@@ -104,14 +104,17 @@ class MiGA::DistanceRunner
     # Calculate all the AAIs/ANIs against the lowest subclade (if classified)
     par_dir = File.dirname(File.expand_path(classif, res.dir))
     par = File.expand_path('miga-project.classif', par_dir)
-    closest = {dataset: nil, ani: 0.0}
+    closest = { dataset: nil, ani: 0.0 }
     if File.size? par
       File.open(par, 'r') do |fh|
         fh.each_line do |ln|
           r = ln.chomp.split("\t")
           next unless r[1].to_i == val_cls
+
           ani = ani_after_aai(ref_project.dataset(r[0]), 80.0)
-          closest = {ds: r[0], ani: ani} unless ani.nil? or ani < closest[:ani]
+          unless ani.nil? || ani < closest[:ani]
+            closest = { ds: r[0], ani: ani }
+          end
         end
       end
     end
@@ -119,22 +122,23 @@ class MiGA::DistanceRunner
     # Calculate all the AAIs/ANIs against the closest ANI95-clade (if AAI > 80%)
     cl_path = res.file_path :clades_ani95
     if !cl_path.nil? and File.size? cl_path and tsk[0] == :clade_finding
-      File.foreach(cl_path).
-        map  { |i| i.chomp.split(',') }.
-        find( lambda{[]} ){ |i| i.include? closest[:ds] }.
-        each { |i| ani_after_aai(ref_project.dataset(i), 80.0) }
+      File.foreach(cl_path)
+          .map { |i| i.chomp.split(',') }
+          .find(lambda { [] }) { |i| i.include? closest[:ds] }
+          .each { |i| ani_after_aai(ref_project.dataset(i), 80.0) }
     end
 
     # Finalize
-    [:haai, :aai, :ani].each{ |m| checkpoint! m if db_counts[m] > 0 }
+    [:haai, :aai, :ani].each { |m| checkpoint! m if db_counts[m] > 0 }
     build_medoids_tree(tsk[1])
     transfer_taxonomy(tax_test)
   end
 
   # Launch analysis for taxonomy jobs
   def go_taxonomy!
-    $stderr.puts "Launching taxonomy analysis"
+    $stderr.puts 'Launching taxonomy analysis'
     return unless project.metadata[:ref_project]
+
     go_query! # <- yeah, it's actually the same, just different ref_project
   end
 end

data/utils/distance/temporal.rb

@@ -1,9 +1,7 @@
-
 require 'tmpdir'
 require 'zlib'
 
 module MiGA::DistanceRunner::Temporal
-
   # Copy input files to the (local) temporal folder
   def create_temporals
     rf = {
@@ -43,7 +41,7 @@ module MiGA::DistanceRunner::Temporal
   def checkpoint!(metric)
     $stderr.puts "Checkpoint (metric = #{metric})"
     SQLite3::Database.new(tmp_dbs[metric]) do |conn|
-      conn.execute("select count(*) from #{metric==:haai ? :aai : metric}")
+      conn.execute("select count(*) from #{metric == :haai ? :aai : metric}")
     end
     FileUtils.cp(tmp_dbs[metric], dbs[metric])
     @db_counts[metric] = 0

data/utils/distances.rb

@@ -4,6 +4,6 @@ require_relative 'distance/runner.rb'
 
 dataset = ARGV.shift
 project = ARGV.shift
-opts = Hash[ ARGV.map{ |i| i.split("=",2).tap{ |j| j[0] = j[0].to_sym } } ]
+opts = Hash[ARGV.map { |i| i.split("=", 2).tap { |j| j[0] = j[0].to_sym } }]
 runner = MiGA::DistanceRunner.new(dataset, project, opts)
 runner.go!

data/utils/domain-ess-genes.rb

@@ -6,10 +6,10 @@ domain = ARGV.shift
 
 def quality(hsh)
   q = {}
-  q[:found] = hsh.values.map{ |i| i==0 ? 0 : 1 }.inject(:+)
-  q[:multi] = hsh.values.map{ |i| i==0 ? 0 : i-1 }.inject(:+)
-  q[:cmp] = 100.0*q[:found].to_f/hsh.size
-  q[:cnt] = 100.0*q[:multi].to_f/hsh.size
+  q[:found] = hsh.values.map { |i| i == 0 ? 0 : 1 }.inject(:+)
+  q[:multi] = hsh.values.map { |i| i == 0 ? 0 : i - 1 }.inject(:+)
+  q[:cmp] = 100.0 * q[:found].to_f / hsh.size
+  q[:cnt] = 100.0 * q[:multi].to_f / hsh.size
   q
 end
 
@@ -39,7 +39,7 @@ end
 # Find expected genes for domain
 n_dom = Hash[
   `HMM.essential.rb -L -q '-#{domain}' -c '#{collection}'`
-    .chomp.split("\n").map { |i| i.split("\t") }
+        .chomp.split("\n").map { |i| i.split("\t") }
 ]
 l_dom = n_dom.keys
 cnt_dom = {}
@@ -54,10 +54,10 @@ File.open(outlog, 'w') do |ofh|
   ofh.puts "! Contamination: #{q[:cnt].round(1)}%."
   if q[:multi] > 0
     ofh.puts "! Multiple copies: "
-    cnt_dom.each{ |k,v| ofh.puts "!   #{v} #{k}: #{n_dom[k]}." if v>1 }
+    cnt_dom.each { |k, v| ofh.puts "!   #{v} #{k}: #{n_dom[k]}." if v > 1 }
   end
   if q[:found] < cnt_dom.size
     ofh.puts "! Missing genes: "
-    cnt_dom.each{ |k,v| ofh.puts "!   #{k}: #{n_dom[k]}." if v==0 }
+    cnt_dom.each { |k, v| ofh.puts "!   #{k}: #{n_dom[k]}." if v == 0 }
   end
 end