miga-base 0.7.23.0 → 0.7.25.3

data/test/lair_test.rb

@@ -75,10 +75,9 @@ class LairTest < Test::Unit::TestCase
       assert_predicate(d, :active?)
     end
 
-    out = capture_stderr { lair.terminate_daemons }.string
+    out = capture_stderr { lair.terminate_daemons ; sleep(2) }.string
     assert_match(/Probing MiGA::Daemon/, out)
     assert_match(/Sending termination message/, out)
-    sleep(2)
     assert_not_predicate(d, :active?)
 
     out = capture_stderr { assert { lair.daemon_loop } }.string

data/test/result_test.rb

@@ -38,6 +38,28 @@ class ResultTest < Test::Unit::TestCase
     assert_instance_of(MiGA::Result, r)
   end
 
+  def test_unlink
+    r = project.add_result(:clade_finding)
+    path = r.path
+    done = r.path(:done)
+    data = r.file_path(:empty)
+    assert(File.exist?(path))
+    assert(File.exist?(done))
+    assert(File.exist?(data))
+    r.unlink
+    assert(!File.exist?(path))
+    assert(!File.exist?(done))
+    assert(File.exist?(data))
+  end
+
+  def test_remove
+    r = project.add_result(:clade_finding)
+    data = r.file_path(:empty)
+    assert(File.exist?(data))
+    r.remove!
+    assert(!File.exist?(data))
+  end
+
   def test_result_source
     r = dataset.add_result(:trimmed_reads)
     assert_equal(dataset.name, r.source.name)

data/test/with_option_test.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+require 'miga/common/with_option'
+
+class WithDaemonTest < Test::Unit::TestCase
+  include TestHelper
+
+  class TestWithOption < MiGA::MiGA
+    include MiGA::Common::WithOption
+
+    attr_reader :metadata, :saved
+
+    def initialize
+      @metadata = { range: 0.9 }
+      @saved = false
+    end
+
+    def self.OPTIONS
+      {
+        empty: {},
+        float: { type: Float },
+        range: { default: 1.0, in: -5.5..5.5, type: Float },
+        default: { default: 9, type: Integer },
+        token: { type: Integer, tokens: %w[yes no 0] },
+        proc: { default: proc { Date.today } },
+        bool: { in: [true, false] }
+      }
+    end
+
+    def save
+      @saved = true
+    end
+  end
+
+  def test_with_option
+    o = TestWithOption.new
+    assert_respond_to(o, :option)
+    assert_equal(1, o.metadata.size)
+  end
+
+  def test_option
+    o = TestWithOption.new
+    assert_equal(9, o.option(:default))
+    assert_nil(o.option(:bool))
+    assert_raise { o.option(:not_an_option) }
+    assert_nil(o.option(:empty))
+  end
+
+  def test_set_bool
+    o = TestWithOption.new
+    assert_nil(o.option(:bool))
+    assert(!o.saved)
+    assert_raise { o.set_option(:bool, 'true') }
+    assert_nil(o.option(:bool))
+    assert(!o.saved)
+    assert_equal(true, o.set_option(:bool, 'true', true))
+    assert(o.saved)
+    assert_equal(false, o.set_option(:bool, false))
+    assert_equal(false, o.set_option(:bool, 'false', true))
+    assert_nil(o.set_option(:bool, nil))
+  end
+
+  def test_set_empty
+    o = TestWithOption.new
+    assert_nil(o.option(:empty))
+    assert_equal('a', o.set_option(:empty, 'a'))
+    assert_equal('1', o.set_option(:empty, '1', true))
+  end
+
+  def test_all_options
+    o = TestWithOption.new
+    assert(o.all_options.is_a?(Hash))
+    assert_include(o.all_options.keys, :bool)
+    assert_nil(o.all_options[:bool])
+  end
+
+  def test_option?
+    o = TestWithOption.new
+    assert(o.option?(:range))
+    assert(!o.option?(:not_an_option))
+  end
+
+  def test_option_metadata
+    o = TestWithOption.new
+    assert_equal(0.9, o.option(:range))
+    assert_equal(1.0, o.set_option(:range, nil))
+    assert_equal(2.0, o.set_option(:range, 2.0))
+    assert_equal(3.0, o.set_option(:range, '3', true))
+  end
+
+  def test_option_range
+    o = TestWithOption.new
+    assert_raise { o.set_option(:range, 9.0) }
+    assert_raise { o.set_option(:range, 3) }
+    assert_raise { o.set_option(:range, true) }
+  end
+
+  def test_option_proc
+    o = TestWithOption.new
+    assert(o.option(:proc).is_a?(Date))
+    assert(o.set_option(:proc, 1).is_a?(Integer))
+    assert(o.set_option(:proc, nil).is_a?(Date))
+  end
+
+  def test_token
+    o = TestWithOption.new
+    assert_nil(o.option(:token))
+    assert_equal(1, o.set_option(:token, 1))
+    assert_equal(-2, o.set_option(:token, '-2', true))
+    assert_equal('yes', o.set_option(:token, 'yes'))
+    assert_equal('0', o.set_option(:token, '0', true))
+    assert_raise { o.set_option(:token, 'maybe') }
+  end
+end

data/utils/cleanup-databases.rb

@@ -19,7 +19,7 @@ m.say 'Cleaning Databases'
       next unless (idx % thr) == t
 
       d = p.dataset(i)
-      next unless d.is_ref? and d.is_active?
+      next unless d.ref? && d.active?
 
       d.cleanup_distances!
     end
@@ -28,4 +28,3 @@ end
 Process.waitall
 m.advance('Dataset:', dsn.size, dsn.size)
 m.say
-

data/utils/distance/base.rb

@@ -2,4 +2,13 @@ require 'miga'
 require 'miga/tax_dist'
 
 class MiGA::DistanceRunner
+  require_relative 'temporal.rb'
+  require_relative 'database.rb'
+  require_relative 'commands.rb'
+  require_relative 'pipeline.rb'
+
+  include MiGA::DistanceRunner::Temporal
+  include MiGA::DistanceRunner::Database
+  include MiGA::DistanceRunner::Commands
+  include MiGA::DistanceRunner::Pipeline
 end

data/utils/distance/commands.rb

@@ -1,105 +1,207 @@
 module MiGA::DistanceRunner::Commands
-  # Estimates or calculates AAI against +target+
-  def aai(target)
-    # Check if the request makes sense
-    return nil if target.nil? || target.result(:essential_genes).nil?
-
-    # Check if it's been calculated
-    y = stored_value(target, :aai)
-    return y unless y.nil? || y.zero?
-
-    # Try hAAI (except in clade projects)
-    unless @ref_project.is_clade?
-      y = haai(target)
-      return y unless y.nil? || y.zero?
+  ##
+  # Estimates AAI against +targets+ using hAAI
+  def haai(targets)
+    puts "[#{Time.now}] hAAI: #{dataset.name} vs #{targets.size} targets"
+    empty_vals = targets.map { |_i| nil }
+    return empty_vals if opts[:haai_p] == 'no'
+
+    # Launch comparisons
+    sbj = pending_targets(targets, :haai)
+    unless sbj.empty?
+      opts[:haai_p] == 'fastaai' ? fastaai_cmd(sbj) : haai_cmd(sbj)
     end
-    # Full AAI
-    aai_cmd(
-      tmp_file('proteins.fa'), target.result(:cds).file_path(:proteins),
-      dataset.name, target.name, tmp_dbs[:aai]
-    ).tap { checkpoint :aai }
+
+    # Return AAI estimates from the database
+    batch_values_from_db(:aai, targets.map { |i| i&.name })
   end
 
   ##
-  # Estimates AAI against +target+ using hAAI
-  def haai(target)
-    return nil if opts[:haai_p] == 'no'
-
-    haai = aai_cmd(tmp_file('ess_genes.fa'),
-                   target.result(:essential_genes).file_path(:ess_genes),
-                   dataset.name, target.name, tmp_dbs[:haai],
-                   aai_save_rbm: 'no-save-rbm', aai_p: opts[:haai_p])
-    checkpoint :haai
-    return nil if haai.nil? || haai.zero? || haai > 90.0
-
-    aai = 100.0 - Math.exp(2.435076 + 0.4275193 * Math.log(100.0 - haai))
-    SQLite3::Database.new(tmp_dbs[:aai]) do |conn|
-      conn.execute 'insert into aai values(?, ?, ?, 0, 0, 0)',
-                   [dataset.name, target.name, aai]
+  # Estimates or calculates AAI against +targets+
+  def aai(targets)
+    puts "[#{Time.now}] AAI: #{dataset.name} vs #{targets.size} targets"
+
+    # Try hAAI first
+    haai(targets)
+
+    # Launch comparisons
+    pending_targets(targets, :aai).each do |target|
+      # Full AAI
+      target_cds = target.result(:cds).file_path(:proteins) or next
+      aairb_cmd(
+        tmp_file('proteins.fa'), target_cds,
+        dataset.name, target.name, tmp_dbs[:aai], checkpoint: :aai
+      )
     end
-    checkpoint :aai
-    aai
+
+    # Return AAI from the database
+    batch_values_from_db(:aai, targets.map { |i| i&.name })
   end
 
   ##
-  # Calculates ANI against +target+
-  def ani(target)
-    # Check if the request makes sense
-    t = tmp_file('largecontigs.fa')
-    r = target.result(:assembly)
-    return nil if r.nil? || !File.size?(t)
-
-    # Check if it's been calculated
-    y = stored_value(target, :ani)
-    return y unless y.nil? || y.zero?
-
-    # Run it
-    ani_cmd(
-      t, r.file_path(:largecontigs),
-      dataset.name, target.name, tmp_dbs[:ani]
-    ).tap { checkpoint :ani }
+  # Calculates ANI against +targets+
+  def ani(targets)
+    puts "[#{Time.now}] ANI: #{dataset.name} vs #{targets.size} targets"
+    empty_vals = targets.map { |_i| nil }
+    return empty_vals unless File.size?(tmp_file('largecontigs.fa'))
+
+    # Launch comparisons
+    sbj = pending_targets(targets, :ani)
+    unless sbj.empty?
+      opts[:ani_p] == 'fastani' ? fastani_cmd(sbj) : anirb_cmd(sbj)
+    end
+
+    # Return ANI from the database
+    batch_values_from_db(:ani, targets.map { |i| i&.name })
   end
 
   ##
-  # Calculates and returns ANI against +target+ if AAI >= +aai_limit+.
-  # Returns +nil+ otherwise
-  def ani_after_aai(target, aai_limit = 85.0)
-    aai = aai(target)
-    (aai.nil? || aai < aai_limit) ? nil : ani(target)
+  # Calculates and returns ANI against +targets+ if AAI >= +aai_limit+.
+  # Note that ANI values may be returned for lower (or failing) AAIs if the
+  # value is already stored in the database
+  def ani_after_aai(targets, aai_limit = 85.0)
+    # Run AAI and select targets with AAI ≥ aai_limit
+    aai = aai(targets)
+    sbj = aai.each_with_index.map { |i, k| targets[k] if i&.> aai_limit }
+    sbj.compact!
+
+    # Run ANI
+    ani(sbj) unless sbj.empty?
+
+    # Return ANI from the database
+    batch_values_from_db(:ani, targets.map { |i| i&.name })
   end
 
   ##
   # Execute an AAI command
-  def aai_cmd(f1, f2, n1, n2, db, o = {})
+  def aairb_cmd(f1, f2, n1, n2, db, o = {})
     o = opts.merge(o)
-    v = `aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
-          --name1 "#{n1}" --name2 "#{n2}" \
-          -t "#{o[:thr]}" -a --lookup-first "--#{o[:aai_save_rbm]}" \
-          -p "#{o[:aai_p] || 'blast+'}"`.chomp
-    (v.nil? || v.empty?) ? 0 : v.to_f
+    run_cmd <<~CMD
+              aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
+              --name1 "#{n1}" --name2 "#{n2}" \
+              -t "#{o[:thr]}" -a --#{'no-' unless o[:aai_save_rbm]}save-rbm \
+              -p "#{o[:aai_p]}"
+            CMD
+  ensure
+    checkpoint(o[:checkpoint]) if o[:checkpoint]
   end
 
   ##
-  # Execute an ANI command
-  def ani_cmd(f1, f2, n1, n2, db, o = {})
-    o = opts.merge(o)
-    v = nil
-    if o[:ani_p] == 'fastani'
-      out = `fastANI -r "#{f1}" -q "#{f2}" \
-            -o /dev/stdout 2>/dev/null`.chomp.split(/\s+/)
-      unless out.empty?
-        SQLite3::Database.new(db) do |conn|
-          conn.execute 'insert into ani values(?, ?, ?, 0, ?, ?)',
-                       [n1, n2, out[2], out[3], out[4]]
-        end
+  # Execute an ani.rb command
+  def anirb_cmd(targets)
+    f1 = tmp_file('largecontigs.fa')
+    return unless File.size?(f1)
+
+    targets.each do |target|
+      target_asm = target&.result(:assembly)&.file_path(:largecontigs) or next
+      run_cmd <<~CMD
+                ani.rb -1 "#{f1}" -2 "#{target_asm}" -S "#{tmp_dbs[:ani]}" \
+                --name1 "#{dataset.name}" --name2 "#{target.name}" \
+                -t "#{opts[:thr]}" -a --no-save-regions --no-save-rbm \
+                -p "#{opts[:ani_p]}"
+              CMD
+      checkpoint(:ani)
+    end
+  end
+
+  ##
+  # Execute a FastANI command
+  def fastani_cmd(targets)
+    f1 = tmp_file('largecontigs.fa')
+    return unless File.size?(f1)
+
+    # Run FastANI
+    File.open(f2 = tmp_file, 'w') do |fh|
+      targets.each do |target|
+        target_asm = target&.result(:assembly)&.file_path(:largecontigs)
+        fh.puts target_asm if target_asm
+      end
+    end
+    run_cmd <<~CMD
+              fastANI -q "#{f1}" --rl "#{f2}" -t #{opts[:thr]} \
+              -o "#{f3 = tmp_file}"
+            CMD
+
+    # Retrieve resulting data and save to DB
+    data = {}
+    File.open(f3, 'r') do |fh|
+      fh.each do |ln|
+        row = ln.chomp.split("\t")
+        n2 = File.basename(row[1], '.gz')
+        n2 = File.basename(n2, '.LargeContigs.fna')
+        data[n2] = [row[2].to_f, 0.0, row[3].to_i, row[4].to_i]
+      end
+    end
+    batch_data_to_db(:ani, data)
+
+    # Cleanup
+    [f2, f3].each { |i| File.unlink(i) }
+  end
+
+  ##
+  # Execute a FastAAI command
+  def fastaai_cmd(targets)
+    qry_idx = dataset.result(:essential_genes).file_path(:fastaai_index)
+    return nil unless qry_idx
+
+    # Run FastAAI
+    File.open(f1 = tmp_file, 'w') { |fh| fh.puts qry_idx }
+    File.open(f2 = tmp_file, 'w') do |fh|
+      targets.each do |target|
+        target_idx = target&.result(:essential_genes)&.file_path(:fastaai_index)
+        fh.puts target_idx if target_idx
+      end
+    end
+    run_cmd <<~CMD
+              FastAAI --qd "#{f1}" --rd "#{f2}" --output "#{f3 = tmp_file}" \
+              --threads #{opts[:thr]}
+            CMD
+
+    # Save values in the databases
+    haai_data = {}
+    aai_data = {}
+    File.open(f3, 'r') do |fh|
+      fh.each do |ln|
+        out = ln.chomp.split("\t")
+        haai_data[out[1]] = [
+          out[2].to_f * 100, out[3].to_f * 100, out[4].to_i, out[5].to_i
+        ]
+        aai_data[out[1]] = [out[6].to_f, 0, 0, 0] if out[6] !~ /^>/
       end
-      v = out[2]
-    else
-      v = `ani.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
-            --name1 "#{n1}" --name2 "#{n2}" \
-            -t "#{opts[:thr]}" -a --no-save-regions --no-save-rbm \
-            --lookup-first -p "#{o[:ani_p] || 'blast+'}"`.chomp
     end
-    v.nil? || v.empty? ? 0 : v.to_f
+    batch_data_to_db(:haai, haai_data)
+    batch_data_to_db(:aai, aai_data)
+
+    # Cleanup
+    [f1, f2, f3].each { |i| File.unlink(i) }
+  end
+
+  ##
+  # Execute an hAAI command
+  def haai_cmd(targets)
+    aai_data = {}
+    targets.each do |target|
+      target_ess = target&.result(:essential_genes)&.file_path(:ess_genes)
+      next unless target_ess
+
+      # hAAI
+      h = aairb_cmd(
+            tmp_file('ess_genes.fa'), target_ess,
+            dataset.name, target.name, tmp_dbs[:haai],
+            aai_save_rbm: false, aai_p: opts[:haai_p], checkpoint: :haai
+          )&.chomp&.to_f
+      next if h.nil? || h.zero? || h > 90.0
+
+      # Estimated AAI
+      aai_data[target.name] = [
+        100.0 - Math.exp(2.435076 + 0.4275193 * Math.log(100.0 - h)), 0, 0, 0
+      ] unless h&.zero? || h > 90.0
+    end
+    batch_data_to_db(:aai, aai_data)
+  end
+
+  def run_cmd(cmd)
+    puts "CMD: #{cmd}"
+    `#{cmd}`
   end
 end