miga-base 0.7.22.0 → 0.7.25.0

data/utils/cleanup-databases.rb

@@ -15,11 +15,11 @@ m.say 'Cleaning Databases'
 (0..thr - 1).each do |t|
   fork do
     dsn.each_with_index do |i, idx|
-      m.advance('Dataset:', dsn.size, idx + 1) if t == 0
+      m.advance('Dataset:', idx + 1, dsn.size) if t == 0
       next unless (idx % thr) == t
 
       d = p.dataset(i)
-      next unless d.is_ref? and d.is_active?
+      next unless d.ref? && d.active?
 
       d.cleanup_distances!
     end
@@ -28,4 +28,3 @@ end
 Process.waitall
 m.advance('Dataset:', dsn.size, dsn.size)
 m.say
-

data/utils/distance/base.rb

@@ -2,4 +2,13 @@ require 'miga'
 require 'miga/tax_dist'
 
 class MiGA::DistanceRunner
+  require_relative 'temporal.rb'
+  require_relative 'database.rb'
+  require_relative 'commands.rb'
+  require_relative 'pipeline.rb'
+
+  include MiGA::DistanceRunner::Temporal
+  include MiGA::DistanceRunner::Database
+  include MiGA::DistanceRunner::Commands
+  include MiGA::DistanceRunner::Pipeline
 end

data/utils/distance/commands.rb

@@ -1,105 +1,207 @@
 module MiGA::DistanceRunner::Commands
-  # Estimates or calculates AAI against +target+
-  def aai(target)
-    # Check if the request makes sense
-    return nil if target.nil? || target.result(:essential_genes).nil?
-
-    # Check if it's been calculated
-    y = stored_value(target, :aai)
-    return y unless y.nil? || y.zero?
-
-    # Try hAAI (except in clade projects)
-    unless @ref_project.is_clade?
-      y = haai(target)
-      return y unless y.nil? || y.zero?
+  ##
+  # Estimates AAI against +targets+ using hAAI
+  def haai(targets)
+    puts "[#{Time.now}] hAAI: #{dataset.name} vs #{targets.size} targets"
+    empty_vals = targets.map { |_i| nil }
+    return empty_vals if opts[:haai_p] == 'no'
+
+    # Launch comparisons
+    sbj = pending_targets(targets, :haai)
+    unless sbj.empty?
+      opts[:haai_p] == 'fastaai' ? fastaai_cmd(sbj) : haai_cmd(sbj)
     end
-    # Full AAI
-    aai_cmd(
-      tmp_file('proteins.fa'), target.result(:cds).file_path(:proteins),
-      dataset.name, target.name, tmp_dbs[:aai]
-    ).tap { checkpoint :aai }
+
+    # Return AAI estimates from the database
+    batch_values_from_db(:aai, targets.map { |i| i&.name })
   end
 
   ##
-  # Estimates AAI against +target+ using hAAI
-  def haai(target)
-    return nil if opts[:haai_p] == 'no'
-
-    haai = aai_cmd(tmp_file('ess_genes.fa'),
-                   target.result(:essential_genes).file_path(:ess_genes),
-                   dataset.name, target.name, tmp_dbs[:haai],
-                   aai_save_rbm: 'no-save-rbm', aai_p: opts[:haai_p])
-    checkpoint :haai
-    return nil if haai.nil? || haai.zero? || haai > 90.0
-
-    aai = 100.0 - Math.exp(2.435076 + 0.4275193 * Math.log(100.0 - haai))
-    SQLite3::Database.new(tmp_dbs[:aai]) do |conn|
-      conn.execute 'insert into aai values(?, ?, ?, 0, 0, 0)',
-                   [dataset.name, target.name, aai]
+  # Estimates or calculates AAI against +targets+
+  def aai(targets)
+    puts "[#{Time.now}] AAI: #{dataset.name} vs #{targets.size} targets"
+
+    # Try hAAI first
+    haai(targets)
+
+    # Launch comparisons
+    pending_targets(targets, :aai).each do |target|
+      # Full AAI
+      target_cds = target.result(:cds).file_path(:proteins) or next
+      aairb_cmd(
+        tmp_file('proteins.fa'), target_cds,
+        dataset.name, target.name, tmp_dbs[:aai], checkpoint: :aai
+      )
     end
-    checkpoint :aai
-    aai
+
+    # Return AAI from the database
+    batch_values_from_db(:aai, targets.map { |i| i&.name })
   end
 
   ##
-  # Calculates ANI against +target+
-  def ani(target)
-    # Check if the request makes sense
-    t = tmp_file('largecontigs.fa')
-    r = target.result(:assembly)
-    return nil if r.nil? || !File.size?(t)
-
-    # Check if it's been calculated
-    y = stored_value(target, :ani)
-    return y unless y.nil? || y.zero?
-
-    # Run it
-    ani_cmd(
-      t, r.file_path(:largecontigs),
-      dataset.name, target.name, tmp_dbs[:ani]
-    ).tap { checkpoint :ani }
+  # Calculates ANI against +targets+
+  def ani(targets)
+    puts "[#{Time.now}] ANI: #{dataset.name} vs #{targets.size} targets"
+    empty_vals = targets.map { |_i| nil }
+    return empty_vals unless File.size?(tmp_file('largecontigs.fa'))
+
+    # Launch comparisons
+    sbj = pending_targets(targets, :ani)
+    unless sbj.empty?
+      opts[:ani_p] == 'fastani' ? fastani_cmd(sbj) : anirb_cmd(sbj)
+    end
+
+    # Return ANI from the database
+    batch_values_from_db(:ani, targets.map { |i| i&.name })
   end
 
   ##
-  # Calculates and returns ANI against +target+ if AAI >= +aai_limit+.
-  # Returns +nil+ otherwise
-  def ani_after_aai(target, aai_limit = 85.0)
-    aai = aai(target)
-    (aai.nil? || aai < aai_limit) ? nil : ani(target)
+  # Calculates and returns ANI against +targets+ if AAI >= +aai_limit+.
+  # Note that ANI values may be returned for lower (or failing) AAIs if the
+  # value is already stored in the database
+  def ani_after_aai(targets, aai_limit = 85.0)
+    # Run AAI and select targets with AAI ≥ aai_limit
+    aai = aai(targets)
+    sbj = aai.each_with_index.map { |i, k| targets[k] if i&.> aai_limit }
+    sbj.compact!
+
+    # Run ANI
+    ani(sbj) unless sbj.empty?
+
+    # Return ANI from the database
+    batch_values_from_db(:ani, targets.map { |i| i&.name })
   end
 
   ##
   # Execute an AAI command
-  def aai_cmd(f1, f2, n1, n2, db, o = {})
+  def aairb_cmd(f1, f2, n1, n2, db, o = {})
     o = opts.merge(o)
-    v = `aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
-          --name1 "#{n1}" --name2 "#{n2}" \
-          -t "#{o[:thr]}" -a --lookup-first "--#{o[:aai_save_rbm]}" \
-          -p "#{o[:aai_p] || 'blast+'}"`.chomp
-    (v.nil? || v.empty?) ? 0 : v.to_f
+    run_cmd <<~CMD
+              aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
+              --name1 "#{n1}" --name2 "#{n2}" \
+              -t "#{o[:thr]}" -a --#{'no-' unless o[:aai_save_rbm]}save-rbm \
+              -p "#{o[:aai_p]}"
+            CMD
+  ensure
+    checkpoint(o[:checkpoint]) if o[:checkpoint]
   end
 
   ##
-  # Execute an ANI command
-  def ani_cmd(f1, f2, n1, n2, db, o = {})
-    o = opts.merge(o)
-    v = nil
-    if o[:ani_p] == 'fastani'
-      out = `fastANI -r "#{f1}" -q "#{f2}" \
-            -o /dev/stdout 2>/dev/null`.chomp.split(/\s+/)
-      unless out.empty?
-        SQLite3::Database.new(db) do |conn|
-          conn.execute 'insert into ani values(?, ?, ?, 0, ?, ?)',
-                       [n1, n2, out[2], out[3], out[4]]
-        end
+  # Execute an ani.rb command
+  def anirb_cmd(targets)
+    f1 = tmp_file('largecontigs.fa')
+    return unless File.size?(f1)
+
+    targets.each do |target|
+      target_asm = target&.result(:assembly)&.file_path(:largecontigs) or next
+      run_cmd <<~CMD
+                ani.rb -1 "#{f1}" -2 "#{target_asm}" -S "#{tmp_dbs[:ani]}" \
+                --name1 "#{dataset.name}" --name2 "#{target.name}" \
+                -t "#{opts[:thr]}" -a --no-save-regions --no-save-rbm \
+                -p "#{opts[:ani_p]}"
+              CMD
+      checkpoint(:ani)
+    end
+  end
+
+  ##
+  # Execute a FastANI command
+  def fastani_cmd(targets)
+    f1 = tmp_file('largecontigs.fa')
+    return unless File.size?(f1)
+
+    # Run FastANI
+    File.open(f2 = tmp_file, 'w') do |fh|
+      targets.each do |target|
+        target_asm = target&.result(:assembly)&.file_path(:largecontigs)
+        fh.puts target_asm if target_asm
+      end
+    end
+    run_cmd <<~CMD
+              fastANI -q "#{f1}" --rl "#{f2}" -t #{opts[:thr]} \
+              -o "#{f3 = tmp_file}"
+            CMD
+
+    # Retrieve resulting data and save to DB
+    data = {}
+    File.open(f3, 'r') do |fh|
+      fh.each do |ln|
+        row = ln.chomp.split("\t")
+        n2 = File.basename(row[1], '.gz')
+        n2 = File.basename(n2, '.LargeContigs.fna')
+        data[n2] = [row[2].to_f, 0.0, row[3].to_i, row[4].to_i]
+      end
+    end
+    batch_data_to_db(:ani, data)
+
+    # Cleanup
+    [f2, f3].each { |i| File.unlink(i) }
+  end
+
+  ##
+  # Execute a FastAAI command
+  def fastaai_cmd(targets)
+    qry_idx = dataset.result(:essential_genes).file_path(:fastaai_index)
+    return nil unless qry_idx
+
+    # Run FastAAI
+    File.open(f1 = tmp_file, 'w') { |fh| fh.puts qry_idx }
+    File.open(f2 = tmp_file, 'w') do |fh|
+      targets.each do |target|
+        target_idx = target&.result(:essential_genes)&.file_path(:fastaai_index)
+        fh.puts target_idx if target_idx
+      end
+    end
+    run_cmd <<~CMD
+              FastAAI --qd "#{f1}" --rd "#{f2}" --output "#{f3 = tmp_file}" \
+              --threads #{opts[:thr]}
+            CMD
+
+    # Save values in the databases
+    haai_data = {}
+    aai_data = {}
+    File.open(f3, 'r') do |fh|
+      fh.each do |ln|
+        out = ln.chomp.split("\t")
+        haai_data[out[1]] = [
+          out[2].to_f * 100, out[3].to_f * 100, out[4].to_i, out[5].to_i
+        ]
+        aai_data[out[1]] = [out[6].to_f, 0, 0, 0] if out[6] !~ /^>/
       end
-      v = out[2]
-    else
-      v = `ani.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
-            --name1 "#{n1}" --name2 "#{n2}" \
-            -t "#{opts[:thr]}" -a --no-save-regions --no-save-rbm \
-            --lookup-first -p "#{o[:ani_p] || 'blast+'}"`.chomp
     end
-    v.nil? || v.empty? ? 0 : v.to_f
+    batch_data_to_db(:haai, haai_data)
+    batch_data_to_db(:aai, aai_data)
+
+    # Cleanup
+    [f1, f2, f3].each { |i| File.unlink(i) }
+  end
+
+  ##
+  # Execute an hAAI command
+  def haai_cmd(targets)
+    aai_data = {}
+    targets.each do |target|
+      target_ess = target&.result(:essential_genes)&.file_path(:ess_genes)
+      next unless target_ess
+
+      # hAAI
+      h = aairb_cmd(
+            tmp_file('ess_genes.fa'), target_ess,
+            dataset.name, target.name, tmp_dbs[:haai],
+            aai_save_rbm: false, aai_p: opts[:haai_p], checkpoint: :haai
+          )&.chomp&.to_f
+      next if h.nil? || h.zero? || h > 90.0
+
+      # Estimated AAI
+      aai_data[target.name] = [
+        100.0 - Math.exp(2.435076 + 0.4275193 * Math.log(100.0 - h)), 0, 0, 0
+      ] unless h&.zero? || h > 90.0
+    end
+    batch_data_to_db(:aai, aai_data)
+  end
+
+  def run_cmd(cmd)
+    puts "CMD: #{cmd}"
+    `#{cmd}`
   end
 end

data/utils/distance/database.rb

@@ -22,12 +22,16 @@ module MiGA::DistanceRunner::Database
         end
       end
       # Initialize if it doesn't exist
-      SQLite3::Database.new(dbs[m]) do |conn|
-        conn.execute "create table if not exists #{t}(" +
-                     "seq1 varchar(256), seq2 varchar(256), " +
-                     "#{t} float, sd float, n int, omega int" +
-                     ")"
-      end unless File.size? dbs[m]
+      unless File.size? dbs[m]
+        SQLite3::Database.new(dbs[m]) do |conn|
+          conn.execute <<~SQL
+            create table if not exists #{t}(
+              seq1 varchar(256), seq2 varchar(256),
+              #{t} float, sd float, n int, omega int
+            )
+          SQL
+        end
+      end
       # Copy over to (local) temporals
       @tmp_dbs[m] = tmp_file("#{m}.db")
       FileUtils.cp(dbs[m], tmp_dbs[m])
@@ -66,7 +70,7 @@ module MiGA::DistanceRunner::Database
     return y unless y.nil? || y.zero?
 
     # Check if self.dataset <- target is done (another thread)
-    if dataset.is_ref? && project.path == ref_project.path
+    if dataset.ref? && project.path == ref_project.path
       y = data_from_db(
         target.name, dataset.name, ref_db(metric, target.name), metric
       )
@@ -92,27 +96,72 @@ module MiGA::DistanceRunner::Database
   # possible number of matches
   def data_from_db(n1, n2, db, metric)
     y = nil
+    table = metric == :haai ? :aai : metric
     SQLite3::Database.new(db) do |conn|
       y = conn.execute(
-        "select #{metric}, sd, n, omega from #{metric} where seq1=? and seq2=?",
+        "select #{table}, sd, n, omega from #{table} where seq1=? and seq2=?",
         [n1, n2]
       ).first
-    end if File.size? db
+    end if File.size?(db)
     y
   end
 
   ##
   # Save +data+ of +metric+ between +n1+ and +n2+ in the +db+ database.
   def data_to_db(n1, n2, db, metric, data)
+    table = metric == :haai ? :aai : metric
     SQLite3::Database.new(db) do |conn|
       conn.execute(
-        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+        "insert into #{table} (seq1, seq2, #{table}, sd, n, omega) " +
         "values (?, ?, ?, ?, ?, ?)", [n1, n2] + data
       )
     end
     checkpoint metric
   end
 
+  ##
+  # Saves +data+ of +metric+ in batch to the temporary database,
+  # and assumes query is +#dataset+. +data+ must be a hash with target names
+  # as key and arrays as values with: [val, sd, n, omega]
+  def batch_data_to_db(metric, data)
+    db = tmp_dbs[metric]
+    table = metric == :haai ? :aai : metric
+    `cp #{db} ~/here.db`
+    SQLite3::Database.new(db) do |conn|
+      data.each do |k, v|
+        sql = <<~SQL
+                insert into #{table} (
+                  seq1, seq2, #{table}, sd, n, omega
+                ) values (?, ?, ?, ?, ?, ?)
+              SQL
+        conn.execute(sql, [dataset.name, k] + v)
+      end
+    end
+    checkpoint(metric)
+  end
+
+  ##
+  # Retrieves data of +metric+ in batch from the temporary database,
+  # and assumes query is +#dataset+. The output data is a hash with the same
+  # structure described for +#batch_data_to_db+
+  def batch_data_from_db(metric)
+    db = tmp_dbs[metric]
+    table = metric == :haai ? :aai : metric
+    data = {}
+    SQLite3::Database.new(db) do |conn|
+      sql = "select seq2, #{table}, sd, n, omega from #{table}"
+      conn.execute(sql).each { |row| data[row.shift] = row }
+    end
+    data
+  end
+
+  ##
+  # Retrieve only +metric+ values against +names+
+  def batch_values_from_db(metric, names)
+    data = batch_data_from_db(metric)
+    names.map { |i| data[i]&.first }
+  end
+
   ##
   # Iterates for each entry in +db+
   def foreach_in_db(db, metric, &blk)
@@ -120,4 +169,14 @@ module MiGA::DistanceRunner::Database
       conn.execute("select * from #{metric}").each { |r| blk[r] }
     end
   end
+
+  ##
+  # Select only those targets that are not yet stored in either direction
+  def pending_targets(targets, metric)
+    saved = batch_data_from_db(metric).keys
+    targets
+      .compact
+      .select { |i| !saved.include?(i.name) }
+      .select { |i| !stored_value(i, metric)&.> 0.0 }
+  end
 end

data/utils/distance/pipeline.rb

@@ -11,19 +11,14 @@ module MiGA::DistanceRunner::Pipeline
     val_med = ''
     val_cls = nil
     i_n = 0
-    File.open(med, 'r') do |med_fh|
-      med_fh.each_line do |med_ln|
-        i_n += 1
-        med_ln.chomp!
-        val = send(metric, ref_project.dataset(med_ln))
-        if !val.nil? and val >= max_val
-          max_val = val
-          val_med = med_ln
-          val_cls = i_n
-          puts "[#{classif}] New max: #{val_med} (#{val_cls}): #{max_val}"
-        end
-      end
-    end
+    sbj_datasets = File.foreach(med).map { |i| ref_project.dataset(i.chomp) }
+    values = send(metric, sbj_datasets)
+    max_idx = values.map(&:to_f).each_with_index.max[1]
+    max_val = values[max_idx]
+    val_med = sbj_dataset[max_idx].name
+    val_cls = max_idx + 1
+    puts "[#{classif}] New max: #{val_med} (#{val_cls}): #{max_val}"
+
     classif = "#{classif}/miga-project.sc-#{val_cls}"
     result_fh.puts [val_cls, val_med, max_val, classif].join("\t")
     classify(clades, classif, metric, result_fh, val_cls)
@@ -31,12 +26,9 @@ module MiGA::DistanceRunner::Pipeline
 
   # Run distances against datasets listed in metadata's +:dist_req+
   def distances_by_request(metric)
-    return unless dataset.metadata[:dist_req]
-
     $stderr.puts 'Running distances by request'
-    dataset.metadata[:dist_req].each do |target|
-      ds = ref_project.dataset(target) and send(metric, ds)
-    end
+    sbj_datasets = dataset.option(:dist_req).map { |i| ref_project.dataset(i) }
+    send(metric, sbj_datasets)
   end
 
   # Builds a tree with all visited medoids from any classification level
@@ -76,8 +68,10 @@ module MiGA::DistanceRunner::Pipeline
     $stderr.puts "Testing taxonomy | opts = #{opts}"
     # Get taxonomy of closest relative
     from_ref_project = (project != ref_project)
-    res_dir = from_ref_project ?
-      File.expand_path('data/09.distances/05.taxonomy', project.path) : home
+    res_dir =
+      from_ref_project ?
+        File.expand_path('data/09.distances/05.taxonomy', project.path) :
+        home
     Dir.mkdir res_dir unless Dir.exist? res_dir
     File.open(File.expand_path("#{dataset.name}.done", res_dir), 'w') do |fh|
       fh.puts Time.now.to_s
@@ -112,7 +106,7 @@ module MiGA::DistanceRunner::Pipeline
     $stderr.puts 'Transferring taxonomy'
     return if tax.nil?
 
-    pval = (project.metadata[:tax_pvalue] || 0.05).to_f
+    pval = project.option(:tax_pvalue)
     tax_a = tax
             .select { |i| i[1] != '?' && i[2] <= pval }
             .map { |i| i[0, 2].join(':') }