miga-base 0.7.24.0 → 0.7.26.0

data/lib/miga/cli/base.rb

@@ -88,19 +88,30 @@ module MiGA::Cli::Base
   @@EXECS = @@TASK_DESC.keys
 
   @@FILE_REGEXP =
-    %r{^(?:.*/)?(.+?)(\.[A-Z]*([12]|Reads|Contigs))?(\.f[nastq]+)?$}i
+    %r{^(?:.*/)?(.+?)(\.[A-Z]*(Reads|Contigs))?(\.f[nastq]+)?(\.gz)?$}i
+
+  @@PAIRED_FILE_REGEXP =
+    %r{^(?:.*/)?(.+?)(\.[A-Z]*([12]|Reads))?(\.f[nastq]+)?(\.gz)?$}i
 end
 
 class MiGA::Cli < MiGA::MiGA
   include MiGA::Cli::Base
 
   class << self
-    def TASK_DESC; @@TASK_DESC end
+    def TASK_DESC
+      @@TASK_DESC
+    end
 
-    def TASK_ALIAS; @@TASK_ALIAS end
+    def TASK_ALIAS
+      @@TASK_ALIAS
+    end
 
-    def EXECS; @@EXECS end
+    def EXECS
+      @@EXECS
+    end
 
-    def FILE_REGEXP; @@FILE_REGEXP end
+    def FILE_REGEXP(paired = false)
+      paired ? @@PAIRED_FILE_REGEXP : @@FILE_REGEXP
+    end
   end
 end

data/lib/miga/common/with_option.rb

@@ -53,7 +53,7 @@ module MiGA::Common::WithOption
     end
 
     if opt[:in] && !opt[:in].include?(value)
-      raise "Value out of range for #{key}: #{value}, not #{opt[:in]}"
+      raise "Value out of range for #{key}: #{value}, not in #{opt[:in]}"
     end
 
     value

data/lib/miga/dataset/result.rb

@@ -276,7 +276,8 @@ module MiGA::Dataset::Result
       ess_genes: '.ess.faa',
       collection: '.ess',
       report: '.ess/log',
-      alignments: '.ess/proteins.aln'
+      alignments: '.ess/proteins.aln',
+      fastaai_index: '.faix.db.gz'
     )
   end
 

data/lib/miga/project/base.rb

@@ -132,7 +132,7 @@ module MiGA::Project::Base
     haai_p: {
       desc: 'Value of aai.rb -p on hAAI', type: String,
       default: proc { |project| project.clade? ? 'no' : 'blast+' },
-      in: %w[blast+ blast blat diamond no]
+      in: %w[fastaai blast+ blast blat diamond no]
     },
     aai_p: {
       desc: 'Value of aai.rb -p on AAI', default: 'blast+', type: String,

data/lib/miga/result.rb

@@ -1,5 +1,4 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
 
 require 'miga/result/dates'
 require 'miga/result/source'
@@ -29,13 +28,17 @@ class MiGA::Result < MiGA::MiGA
       MiGA::Result.new(path)
     end
 
-    def create(path, force = false, &blk)
+    ##
+    # Check if +path+ describes a result and otherwise create
+    # it using the passed block. If +force+, ignore existing
+    # JSON in +path+ if any.
+    def create(path, force = false)
       FileUtils.rm(path) if force && File.exist?(path)
-      r_pre = self.load(path)
+      r_pre = load(path)
       return r_pre unless r_pre.nil?
 
       yield
-      self.load(path)
+      load(path)
     end
   end
 
@@ -49,7 +52,7 @@ class MiGA::Result < MiGA::MiGA
   # Load or create the MiGA::Result described by the JSON file +path+
   def initialize(path)
     @path = File.absolute_path(path)
-    MiGA::Result.exist?(@path) ? self.load : create
+    MiGA::Result.exist?(@path) ? load : create
   end
 
   ##
@@ -162,7 +165,7 @@ class MiGA::Result < MiGA::MiGA
       File.unlink s
     end
     MiGA::Json.generate(data, path)
-    self.load
+    load
   end
 
   ##
@@ -182,10 +185,9 @@ class MiGA::Result < MiGA::MiGA
   # Unlink result by removing the .done and .start timestamps and the
   # .json descriptor, but don't remove any other associated files
   def unlink
-    %i(start done).each do |i|
-      f = path(i) and File.exists?(f) and File.unlink(f)
+    %i[start done json].each do |i|
+      f = path(i) and File.exist?(f) and File.unlink(f)
     end
-    File.unlink path
   end
 
   ##
@@ -201,13 +203,14 @@ class MiGA::Result < MiGA::MiGA
 
     @data[:files] ||= {}
     self[:files].each do |k, files|
-      files = [files] unless files.kind_of? Array
+      files = [files] unless files.is_a? Array
       files.each do |file|
         case blk.arity
-        when 1; blk.call(file)
-        when 2; blk.call(k, file)
-        when 3; blk.call(k, file, File.expand_path(file, dir))
-        else; raise "Wrong number of arguments: #{blk.arity} for 1..3"
+        when 1 then blk.call(file)
+        when 2 then blk.call(k, file)
+        when 3 then blk.call(k, file, File.expand_path(file, dir))
+        else
+          raise "Wrong number of arguments: #{blk.arity} for 1..3"
         end
       end
     end

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 24, 0].freeze
+  VERSION = [0.7, 26, 0].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 2, 16)
+  VERSION_DATE = Date.new(2021, 3, 1)
 
   ##
   # Reference of MiGA.

data/scripts/essential_genes.bash

@@ -28,13 +28,29 @@ COLL=$(miga option -P "$PROJECT" --key ess_coll)
 if [[ "$TYPE" == "metagenome" || "$TYPE" == "virome" ]] ; then
   FLAGS="--metagenome"
 else
-  FLAGS="--alignments ${DATASET}.ess/proteins.aln"
+  FLAGS=""
 fi
 HMM.essential.rb \
   -i "$FAA" -o "${DATASET}.ess.faa" -m "${DATASET}.ess/" \
   -t "$CORES" -r "$DATASET" --collection "$COLL" $FLAGS \
   > "${DATASET}.ess/log"
 
+# Index for FastAAI
+NOMULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --no-multi \
+            | wc -l | awk '{print $1}')
+if [[ "$NOMULTI" -eq "1" ]] ; then
+  if [[ "$FAA" == *.gz ]] ; then
+    gzip -cd "$FAA" > "${DATASET}.faix"
+  else
+    cp "$FAA" "${DATASET}.faix"
+  fi
+  FastAAI --qp "${DATASET}.faix" --output "${DATASET}.faix" \
+    --ext ".faix" --index --input-paths --all-vs-all --threads "$CORES"
+  rm "${DATASET}.faix"
+  rm "${DATASET}.faix.hmm"
+  rm "${DATASET}.faix.hmm.filt"
+fi
+
 # Reduce files
 if exists "$DATASET".ess/*.faa ; then
   ( cd "${DATASET}.ess" \

data/scripts/miga.bash

@@ -1,12 +1,17 @@
 #!/bin/bash
+
+# Setup environment
 set -e
-#MIGA=${MIGA:-$(cd "$(dirname "$0")/.."; pwd)}
 MIGA_HOME=${MIGA_HOME:-"$HOME"}
+SCRIPT=${SCRIPT:-$(basename "$0" .bash)}
 # shellcheck source=/dev/null
 . "$MIGA_HOME/.miga_rc"
+
+# Ensure submodules are first in PATH
 export PATH="$MIGA/bin:$MIGA/utils/enveomics/Scripts:$PATH"
-SCRIPT=${SCRIPT:-$(basename "$0" .bash)}
+export PATH="$MIGA/utils/FastAAI/FastAAI:$PATH"
 
+# Ancillary functions
 function exists { [[ -e "$1" ]] ; }
 function fx_exists { [[ $(type -t "$1") == "function" ]] ; }
 function miga_start_project_step {
@@ -28,6 +33,7 @@ function miga_end_project_step {
   miga add_result -P "$PROJECT" -r "$SCRIPT" -f
 }
 
+# Environment header
 if [[ "$SCRIPT" != "d" && "$SCRIPT" != "p" ]] ; then
   echo ""
   echo "######[ $SCRIPT ]######"

data/test/lair_test.rb

@@ -75,10 +75,9 @@ class LairTest < Test::Unit::TestCase
       assert_predicate(d, :active?)
     end
 
-    out = capture_stderr { lair.terminate_daemons }.string
+    out = capture_stderr { lair.terminate_daemons ; sleep(2) }.string
     assert_match(/Probing MiGA::Daemon/, out)
     assert_match(/Sending termination message/, out)
-    sleep(2)
     assert_not_predicate(d, :active?)
 
     out = capture_stderr { assert { lair.daemon_loop } }.string

data/test/result_test.rb

@@ -38,6 +38,28 @@ class ResultTest < Test::Unit::TestCase
     assert_instance_of(MiGA::Result, r)
   end
 
+  def test_unlink
+    r = project.add_result(:clade_finding)
+    path = r.path
+    done = r.path(:done)
+    data = r.file_path(:empty)
+    assert(File.exist?(path))
+    assert(File.exist?(done))
+    assert(File.exist?(data))
+    r.unlink
+    assert(!File.exist?(path))
+    assert(!File.exist?(done))
+    assert(File.exist?(data))
+  end
+
+  def test_remove
+    r = project.add_result(:clade_finding)
+    data = r.file_path(:empty)
+    assert(File.exist?(data))
+    r.remove!
+    assert(!File.exist?(data))
+  end
+
   def test_result_source
     r = dataset.add_result(:trimmed_reads)
     assert_equal(dataset.name, r.source.name)

data/utils/distance/base.rb

@@ -2,4 +2,13 @@ require 'miga'
 require 'miga/tax_dist'
 
 class MiGA::DistanceRunner
+  require_relative 'temporal.rb'
+  require_relative 'database.rb'
+  require_relative 'commands.rb'
+  require_relative 'pipeline.rb'
+
+  include MiGA::DistanceRunner::Temporal
+  include MiGA::DistanceRunner::Database
+  include MiGA::DistanceRunner::Commands
+  include MiGA::DistanceRunner::Pipeline
 end

data/utils/distance/commands.rb

@@ -1,105 +1,207 @@
 module MiGA::DistanceRunner::Commands
-  # Estimates or calculates AAI against +target+
-  def aai(target)
-    # Check if the request makes sense
-    return nil if target.nil? || target.result(:essential_genes).nil?
-
-    # Check if it's been calculated
-    y = stored_value(target, :aai)
-    return y unless y.nil? || y.zero?
-
-    # Try hAAI (except in clade projects)
-    unless @ref_project.clade?
-      y = haai(target)
-      return y unless y.nil? || y.zero?
+  ##
+  # Estimates AAI against +targets+ using hAAI
+  def haai(targets)
+    puts "[#{Time.now}] hAAI: #{dataset.name} vs #{targets.size} targets"
+    empty_vals = targets.map { |_i| nil }
+    return empty_vals if opts[:haai_p] == 'no'
+
+    # Launch comparisons
+    sbj = pending_targets(targets, :haai)
+    unless sbj.empty?
+      opts[:haai_p] == 'fastaai' ? fastaai_cmd(sbj) : haai_cmd(sbj)
     end
-    # Full AAI
-    aai_cmd(
-      tmp_file('proteins.fa'), target.result(:cds).file_path(:proteins),
-      dataset.name, target.name, tmp_dbs[:aai]
-    ).tap { checkpoint :aai }
+
+    # Return AAI estimates from the database
+    batch_values_from_db(:aai, targets.map { |i| i&.name })
   end
 
   ##
-  # Estimates AAI against +target+ using hAAI
-  def haai(target)
-    return nil if opts[:haai_p] == 'no'
-
-    haai = aai_cmd(tmp_file('ess_genes.fa'),
-                   target.result(:essential_genes).file_path(:ess_genes),
-                   dataset.name, target.name, tmp_dbs[:haai],
-                   aai_save_rbm: 'no-save-rbm', aai_p: opts[:haai_p])
-    checkpoint :haai
-    return nil if haai.nil? || haai.zero? || haai > 90.0
-
-    aai = 100.0 - Math.exp(2.435076 + 0.4275193 * Math.log(100.0 - haai))
-    SQLite3::Database.new(tmp_dbs[:aai]) do |conn|
-      conn.execute 'insert into aai values(?, ?, ?, 0, 0, 0)',
-                   [dataset.name, target.name, aai]
+  # Estimates or calculates AAI against +targets+
+  def aai(targets)
+    puts "[#{Time.now}] AAI: #{dataset.name} vs #{targets.size} targets"
+
+    # Try hAAI first
+    haai(targets)
+
+    # Launch comparisons
+    pending_targets(targets, :aai).each do |target|
+      # Full AAI
+      target_cds = target.result(:cds).file_path(:proteins) or next
+      aairb_cmd(
+        tmp_file('proteins.fa'), target_cds,
+        dataset.name, target.name, tmp_dbs[:aai], checkpoint: :aai
+      )
     end
-    checkpoint :aai
-    aai
+
+    # Return AAI from the database
+    batch_values_from_db(:aai, targets.map { |i| i&.name })
   end
 
   ##
-  # Calculates ANI against +target+
-  def ani(target)
-    # Check if the request makes sense
-    t = tmp_file('largecontigs.fa')
-    r = target.result(:assembly)
-    return nil if r.nil? || !File.size?(t)
-
-    # Check if it's been calculated
-    y = stored_value(target, :ani)
-    return y unless y.nil? || y.zero?
-
-    # Run it
-    ani_cmd(
-      t, r.file_path(:largecontigs),
-      dataset.name, target.name, tmp_dbs[:ani]
-    ).tap { checkpoint :ani }
+  # Calculates ANI against +targets+
+  def ani(targets)
+    puts "[#{Time.now}] ANI: #{dataset.name} vs #{targets.size} targets"
+    empty_vals = targets.map { |_i| nil }
+    return empty_vals unless File.size?(tmp_file('largecontigs.fa'))
+
+    # Launch comparisons
+    sbj = pending_targets(targets, :ani)
+    unless sbj.empty?
+      opts[:ani_p] == 'fastani' ? fastani_cmd(sbj) : anirb_cmd(sbj)
+    end
+
+    # Return ANI from the database
+    batch_values_from_db(:ani, targets.map { |i| i&.name })
   end
 
   ##
-  # Calculates and returns ANI against +target+ if AAI >= +aai_limit+.
-  # Returns +nil+ otherwise
-  def ani_after_aai(target, aai_limit = 85.0)
-    aai = aai(target)
-    (aai.nil? || aai < aai_limit) ? nil : ani(target)
+  # Calculates and returns ANI against +targets+ if AAI >= +aai_limit+.
+  # Note that ANI values may be returned for lower (or failing) AAIs if the
+  # value is already stored in the database
+  def ani_after_aai(targets, aai_limit = 85.0)
+    # Run AAI and select targets with AAI ≥ aai_limit
+    aai = aai(targets)
+    sbj = aai.each_with_index.map { |i, k| targets[k] if i&.> aai_limit }
+    sbj.compact!
+
+    # Run ANI
+    ani(sbj) unless sbj.empty?
+
+    # Return ANI from the database
+    batch_values_from_db(:ani, targets.map { |i| i&.name })
   end
 
   ##
   # Execute an AAI command
-  def aai_cmd(f1, f2, n1, n2, db, o = {})
+  def aairb_cmd(f1, f2, n1, n2, db, o = {})
     o = opts.merge(o)
-    v = `aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
-          --name1 "#{n1}" --name2 "#{n2}" \
-          -t "#{o[:thr]}" -a --lookup-first "--#{o[:aai_save_rbm]}" \
-          -p "#{o[:aai_p]}"`.chomp
-    (v.nil? || v.empty?) ? 0 : v.to_f
+    run_cmd <<~CMD
+              aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
+              --name1 "#{n1}" --name2 "#{n2}" \
+              -t "#{o[:thr]}" -a --#{'no-' unless o[:aai_save_rbm]}save-rbm \
+              -p "#{o[:aai_p]}"
+            CMD
+  ensure
+    checkpoint(o[:checkpoint]) if o[:checkpoint]
   end
 
   ##
-  # Execute an ANI command
-  def ani_cmd(f1, f2, n1, n2, db, o = {})
-    o = opts.merge(o)
-    v = nil
-    if o[:ani_p] == 'fastani'
-      out = `fastANI -r "#{f1}" -q "#{f2}" \
-            -o /dev/stdout 2>/dev/null`.chomp.split(/\s+/)
-      unless out.empty?
-        SQLite3::Database.new(db) do |conn|
-          conn.execute 'insert into ani values(?, ?, ?, 0, ?, ?)',
-                       [n1, n2, out[2], out[3], out[4]]
-        end
+  # Execute an ani.rb command
+  def anirb_cmd(targets)
+    f1 = tmp_file('largecontigs.fa')
+    return unless File.size?(f1)
+
+    targets.each do |target|
+      target_asm = target&.result(:assembly)&.file_path(:largecontigs) or next
+      run_cmd <<~CMD
+                ani.rb -1 "#{f1}" -2 "#{target_asm}" -S "#{tmp_dbs[:ani]}" \
+                --name1 "#{dataset.name}" --name2 "#{target.name}" \
+                -t "#{opts[:thr]}" -a --no-save-regions --no-save-rbm \
+                -p "#{opts[:ani_p]}"
+              CMD
+      checkpoint(:ani)
+    end
+  end
+
+  ##
+  # Execute a FastANI command
+  def fastani_cmd(targets)
+    f1 = tmp_file('largecontigs.fa')
+    return unless File.size?(f1)
+
+    # Run FastANI
+    File.open(f2 = tmp_file, 'w') do |fh|
+      targets.each do |target|
+        target_asm = target&.result(:assembly)&.file_path(:largecontigs)
+        fh.puts target_asm if target_asm
+      end
+    end
+    run_cmd <<~CMD
+              fastANI -q "#{f1}" --rl "#{f2}" -t #{opts[:thr]} \
+              -o "#{f3 = tmp_file}"
+            CMD
+
+    # Retrieve resulting data and save to DB
+    data = {}
+    File.open(f3, 'r') do |fh|
+      fh.each do |ln|
+        row = ln.chomp.split("\t")
+        n2 = File.basename(row[1], '.gz')
+        n2 = File.basename(n2, '.LargeContigs.fna')
+        data[n2] = [row[2].to_f, 0.0, row[3].to_i, row[4].to_i]
+      end
+    end
+    batch_data_to_db(:ani, data)
+
+    # Cleanup
+    [f2, f3].each { |i| File.unlink(i) }
+  end
+
+  ##
+  # Execute a FastAAI command
+  def fastaai_cmd(targets)
+    qry_idx = dataset.result(:essential_genes).file_path(:fastaai_index)
+    return nil unless qry_idx
+
+    # Run FastAAI
+    File.open(f1 = tmp_file, 'w') { |fh| fh.puts qry_idx }
+    File.open(f2 = tmp_file, 'w') do |fh|
+      targets.each do |target|
+        target_idx = target&.result(:essential_genes)&.file_path(:fastaai_index)
+        fh.puts target_idx if target_idx
+      end
+    end
+    run_cmd <<~CMD
+              FastAAI --qd "#{f1}" --rd "#{f2}" --output "#{f3 = tmp_file}" \
+              --threads #{opts[:thr]}
+            CMD
+
+    # Save values in the databases
+    haai_data = {}
+    aai_data = {}
+    File.open(f3, 'r') do |fh|
+      fh.each do |ln|
+        out = ln.chomp.split("\t")
+        haai_data[out[1]] = [
+          out[2].to_f * 100, out[3].to_f * 100, out[4].to_i, out[5].to_i
+        ]
+        aai_data[out[1]] = [out[6].to_f, 0, 0, 0] if out[6] !~ /^>/
       end
-      v = out[2]
-    else
-      v = `ani.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
-            --name1 "#{n1}" --name2 "#{n2}" \
-            -t "#{opts[:thr]}" -a --no-save-regions --no-save-rbm \
-            --lookup-first -p "#{o[:ani_p] || 'blast+'}"`.chomp
     end
-    v.nil? || v.empty? ? 0 : v.to_f
+    batch_data_to_db(:haai, haai_data)
+    batch_data_to_db(:aai, aai_data)
+
+    # Cleanup
+    [f1, f2, f3].each { |i| File.unlink(i) }
+  end
+
+  ##
+  # Execute an hAAI command
+  def haai_cmd(targets)
+    aai_data = {}
+    targets.each do |target|
+      target_ess = target&.result(:essential_genes)&.file_path(:ess_genes)
+      next unless target_ess
+
+      # hAAI
+      h = aairb_cmd(
+            tmp_file('ess_genes.fa'), target_ess,
+            dataset.name, target.name, tmp_dbs[:haai],
+            aai_save_rbm: false, aai_p: opts[:haai_p], checkpoint: :haai
+          )&.chomp&.to_f
+      next if h.nil? || h.zero? || h > 90.0
+
+      # Estimated AAI
+      aai_data[target.name] = [
+        100.0 - Math.exp(2.435076 + 0.4275193 * Math.log(100.0 - h)), 0, 0, 0
+      ] unless h&.zero? || h > 90.0
+    end
+    batch_data_to_db(:aai, aai_data)
+  end
+
+  def run_cmd(cmd)
+    puts "CMD: #{cmd}"
+    `#{cmd}`
   end
 end