miga-base 0.7.5.0 → 0.7.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6361b203b4612214936255e8b285959cbf556e7e64f88119a058e167774264f9
-  data.tar.gz: 1a3d8df11d57a363a49eecc88011a5337ddb40573cf8727942eea24e5071ecf5
+  metadata.gz: 30d61eb5847a28c4d83a43e6e47ff0738bb819218e677b9aa43de158b441e0ae
+  data.tar.gz: 46338ae15353b71fb6e7eff390c8bb976d1c11fc296bb83f33aab8ffba1a3fa8
 SHA512:
-  metadata.gz: 53786e1830ed8b3c56bffbf1fc581745185084e96b68631323b01fd9994c2b59cdeac582c2f4cef8d0ed622136a2c2cf4f68e06cc3a9e7dbf580d10d1aedc9e5
-  data.tar.gz: 26322daf6a52906466c4f28cc93ef455bac22e98e115fc6f6b3ad685998d58854454c8044126491e5a886910ce5ddceece2713e94acafaf86459442970617f4f
+  metadata.gz: 488e7888039bb9e08e7c257fdd7cb0cf34340766f73b9b46c28b332072f5f207f5dcfb0df08c98b27f5640158d3a14ae6d0f5ab19d78f3e9e1ef44e381d34e74
+  data.tar.gz: 992cfc225eeb2f4b8017260e66c67bc9df39a79757ecdb65e01766079e103385259c1eeb447fdc5e6b769990f96256d384dc4adddf184693a3bc94b8323b9a5a

data/README.md

@@ -41,6 +41,7 @@ Developed and maintained by [Luis M. Rodriguez-R][lrr]. MiGA is the result of a
 collaboration between [Kostas Lab][kostas] at the Georgia Institute of
 Technology and [RDP][rdp] at Michigan State University.
 
+See also the [complete list of contributors](manual/part1/contributors.md).
 
 # License
 

data/lib/miga/cli/action/classify_wf.rb

@@ -42,7 +42,9 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
         '--no-summaries',
         'Do not generate intermediate step summaries'
       ) { |v| cli[:summaries] = v }
-      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)')
+      opts_for_wf(
+        opt, 'Input genome assemblies (nucleotides, FastA)', qual: false
+      )
     end
   end
 

data/lib/miga/cli/action/edit.rb

@@ -17,18 +17,21 @@ class MiGA::Cli::Action::Edit < MiGA::Cli::Action
         'Activate dataset; requires -D'
       ) { |v| cli[:activate] = v }
       opt.on(
-        '--inactivate',
-        'Inactivate dataset; requires -D'
-      ) { |v| cli[:activate] = !v }
+        '--inactivate [reason]',
+        'Inactivate dataset; requires -D',
+        'The argument is optional: reason to inactivate dataset'
+      ) { |v| cli[:activate] = false ; cli[:reason] = v }
     end
   end
 
   def perform
     obj = cli.load_project_or_dataset
     unless cli[:activate].nil?
-      cli.ensure_par({ dataset: '-D' },
-                     '%<name>s is mandatory with --[in-]activate: please provide %<flag>s')
-      cli[:activate] ? obj.activate! : obj.inactivate!
+      cli.ensure_par(
+        { dataset: '-D' },
+        '%<name>s is mandatory with --[in-]activate: please provide %<flag>s'
+      )
+      cli[:activate] ? obj.activate! : obj.inactivate!(cli[:reason])
     end
     cli.add_metadata(obj)
     obj.save

data/lib/miga/cli/action/init.rb

@@ -66,8 +66,10 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
   end
 
   def run_r_cmd(cli, paths, cmd)
-    run_cmd(cli,
-            "echo #{cmd.shellescape} | #{paths['R'].shellescape} --vanilla -q 2>&1")
+    run_cmd(
+      cli,
+      "echo #{cmd.shellescape} | #{paths['R'].shellescape} --vanilla -q 2>&1"
+    )
   end
 
   def test_r_package(cli, paths, pkg)
@@ -81,16 +83,21 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
   end
 
   def test_ruby_gem(cli, paths, pkg)
-    run_cmd(cli,
-            "#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null")
+    run_cmd(
+      cli,
+      "#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null"
+    )
     $?.success?
   end
 
   def install_ruby_gem(cli, paths, pkg)
     gem_cmd = "Gem::GemRunner.new.run %w(install --user #{pkg})"
-    run_cmd(cli, "#{paths['ruby'].shellescape} \
+    run_cmd(
+      cli,
+      "#{paths['ruby'].shellescape} \
         -r rubygems -r rubygems/gem_runner \
-        -e #{gem_cmd.shellescape} 2>&1")
+        -e #{gem_cmd.shellescape} 2>&1"
+    )
   end
 
   def list_requirements
@@ -99,7 +106,7 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
       'no', %w(yes no)
     ) == 'yes'
       cli.puts ''
-      req_path = File.expand_path('utils/requirements.txt', MiGA.root_path)
+      req_path = File.join(MiGA.root_path, 'utils', 'requirements.txt')
       File.open(req_path, 'r') do |fh|
         fh.each_line { |ln| cli.puts ln }
       end
@@ -205,18 +212,18 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
   def check_additional_files(paths)
     if cli[:mytaxa]
       cli.puts 'Looking for MyTaxa databases:'
-      mt = File.dirname paths["MyTaxa"]
+      mt = File.dirname paths['MyTaxa']
       cli.print 'Looking for scores... '
       unless Dir.exist?(File.expand_path('db', mt))
-        cli.puts "no.\nExecute 'python2 #{mt}/utils/download_db.py'."
+        cli.puts "no\nExecute 'python2 #{mt}/utils/download_db.py'"
         exit(1)
       end
-      cli.puts 'yes.'
+      cli.puts 'yes'
       cli.print 'Looking for diamond db... '
       unless File.exist?(File.expand_path('AllGenomes.faa.dmnd', mt))
-        cli.puts "no.\nDownload " \
+        cli.puts "no\nDownload " \
           "'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" \
-          "AllGenomes.faa.dmnd' into #{mt}."
+          "AllGenomes.faa.dmnd' into #{mt}"
         exit(1)
       end
       cli.puts ''
@@ -228,7 +235,7 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     %w(ape cluster vegan).each do |pkg|
       cli.print "Testing #{pkg}... "
       if test_r_package(cli, paths, pkg)
-        cli.puts 'yes.'
+        cli.puts 'yes'
       else
         cli.puts 'no, installing'
         cli.print '' + install_r_package(cli, paths, pkg)
@@ -245,7 +252,7 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     %w(sqlite3 daemons json).each do |pkg|
       cli.print "Testing #{pkg}... "
       if test_ruby_gem(cli, paths, pkg)
-        cli.puts 'yes.'
+        cli.puts 'yes'
       else
         cli.puts 'no, installing'
         # This hackey mess is meant to ensure the test and installation are done

data/lib/miga/cli/action/quality_wf.rb

@@ -15,7 +15,10 @@ class MiGA::Cli::Action::QualityWf < MiGA::Cli::Action
         '-m', '--mytaxa-scan',
         'Perform MyTaxa scan analysis'
       ) { |v| cli[:mytaxa] = v }
-      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)')
+      opts_for_wf(
+        opt, 'Input genome assemblies (nucleotides, FastA)',
+        qual: false
+      )
     end
   end
 

data/lib/miga/cli/action/stats.rb

@@ -14,12 +14,16 @@ class MiGA::Cli::Action::Stats < MiGA::Cli::Action
       ) { |v| cli[:key] = v }
       opt.on(
         '--compute-and-save',
-        'Compute and saves the statistics'
+        'Compute and save the statistics'
       ) { |v| cli[:compute] = v }
       opt.on(
         '--try-load',
         'Check if stat exists instead of computing on --compute-and-save'
       ) { |v| cli[:try_load] = v }
+      opt.on(
+        '--ignore-empty',
+        'If the result does not exist, exit without throwing exceptions'
+      ) { |v| cli[:ignore_result_empty] = v }
     end
   end
 
@@ -27,7 +31,7 @@ class MiGA::Cli::Action::Stats < MiGA::Cli::Action
     if cli[:try_load] && !r[:stats].nil? && !r[:stats].empty?
       cli[:compute] = false
     end
-    r = cli.load_result
+    r = cli.load_result or return
     if cli[:compute]
       cli.say 'Computing statistics'
       r.compute_stats

data/lib/miga/cli/action/summary.rb

@@ -26,6 +26,10 @@ class MiGA::Cli::Action::Summary < MiGA::Cli::Action
         '--with-units',
         'Include units in each cell'
       ) { |v| cli[:units] = v }
+      opt.on(
+        '--compute-and-save',
+        'Compute and save the statistics if not yet available'
+      ) { |v| cli[:compute] = v }
     end
   end
 
@@ -34,7 +38,8 @@ class MiGA::Cli::Action::Summary < MiGA::Cli::Action
     ds = cli.load_and_filter_datasets
     cli.say 'Loading results'
     stats = ds.map do |d|
-      r = d.add_result(cli[:result].to_sym, false)
+      r = d.result(cli[:result])
+      r.compute_stats if cli[:compute] && !r.nil? && r[:stats].empty?
       s = r.nil? ? {} : r[:stats]
       s.tap { |i| i[:dataset] = d.name }
     end

data/lib/miga/cli/action/wf.rb

@@ -15,7 +15,7 @@ module MiGA::Cli::Action::Wf
 
   def opts_for_wf(opt, files_desc, params = {})
     {
-      multi: false, cleanup: true, project_type: false, ncbi: true
+      multi: false, cleanup: true, project_type: false, ncbi: true, qual: true
     }.each { |k, v| params[k] = v if params[k].nil? }
     opt.on(
       '-o', '--out_dir PATH',
@@ -40,6 +40,13 @@ module MiGA::Cli::Action::Wf
         'Only download complete genomes, not drafts'
       ) { |v| cli[:ncbi_draft] = v }
     end
+    if params[:qual]
+      opt.on(
+        '--min-qual FLOAT', Float,
+        'Minimum genome quality to include in analysis',
+        'By default: 50.0'
+      ) { |v| cli[:min_qual] = v }
+    end
     if params[:cleanup]
       opt.on(
         '-c', '--clean',
@@ -89,6 +96,10 @@ module MiGA::Cli::Action::Wf
   end
 
   def opts_for_wf_distances(opt)
+    opt.on('--sensitive', 'Alias to: --aai-p blast+ --ani-p blast+') do
+      cli[:aai_p] = 'blast+'
+      cli[:ani_p] = 'blast+'
+    end
     opt.on('--fast', 'Alias to: --aai-p diamond --ani-p fastani') do
       cli[:aai_p] = 'diamond'
       cli[:ani_p] = 'fastani'
@@ -121,7 +132,7 @@ module MiGA::Cli::Action::Wf
              ]) unless MiGA::Project.exist? cli[:outdir]
     # Define project metadata
     p = cli.load_project(:outdir, '-o')
-    [:haai_p, :aai_p, :ani_p, :ess_coll].each { |i| p_metadata[i] = cli[i] }
+    %i[haai_p aai_p ani_p ess_coll min_qual].each { |i| p_metadata[i] = cli[i] }
     p_metadata[:type] = cli[:project_type]
     transfer_metadata(p, p_metadata)
     # Download datasets
@@ -155,7 +166,7 @@ module MiGA::Cli::Action::Wf
                  '-P', cli[:outdir],
                  '-r', r,
                  '-o', File.expand_path("#{r}.tsv", cli[:outdir]),
-                 '--tab'
+                 '--tab', '--ref', '--active'
                ])
     end
   end

data/lib/miga/cli/objects_helper.rb

@@ -80,7 +80,9 @@ module MiGA::Cli::ObjectsHelper
       raise "Unsupported result for #{klass}: #{self[:result]}"
     end
     r = obj.add_result(self[:result], false)
-    raise "Cannot load result: #{self[:result]}" if r.nil?
+    if r.nil? && !self[:ignore_result_empty]
+      raise "Cannot load result: #{self[:result]}"
+    end
 
     @objects[:result] = r
   end

data/lib/miga/common/format.rb

@@ -90,7 +90,8 @@ module MiGA::Common::Format
     end
     fh.close
 
-    o = { n: l.size, tot: l.inject(:+), max: l.max }
+    o = { n: l.size, tot: l.inject(0, :+), max: l.max }
+    return o if o[:tot].zero?
     o[:avg] = o[:tot].to_f / l.size
     o[:var] = l.map { |a| a**2 }.inject(:+).to_f / l.size - o[:avg]**2
     o[:sd]  = Math.sqrt o[:var]
@@ -106,7 +107,8 @@ module MiGA::Common::Format
         break if pos >= thr
       end
       o[:med] = o[:n].even? ?
-            0.5 * l[o[:n] / 2 - 1, 2].inject(:+) : l[(o[:n] - 1) / 2]
+            0.5 * l[o[:n] / 2 - 1, 2].inject(:+) :
+            l[(o[:n] - 1) / 2]
     end
     o
   end

data/lib/miga/daemon.rb

@@ -72,6 +72,7 @@ class MiGA::Daemon < MiGA::MiGA
     say '-----------------------------------'
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
+    recalculate_status!
     load_status
     say 'Configuration options:'
     say @runopts.to_s
@@ -99,6 +100,7 @@ class MiGA::Daemon < MiGA::MiGA
   end
 
   def recalculate_status!
+    say 'Recalculating status for all datasets'
     project.each_dataset(&:recalculate_status)
   end
 
@@ -158,8 +160,8 @@ class MiGA::Daemon < MiGA::MiGA
   end
 
   ##
-  # Traverse datasets, and returns boolean indicating if at any datasets
-  # are incomplete
+  # Traverse datasets, and returns boolean indicating if at any reference
+  # datasets are incomplete
   def check_datasets
     l_say(2, 'Checking datasets')
     o = false
@@ -167,7 +169,7 @@ class MiGA::Daemon < MiGA::MiGA
       next unless ds.status == :incomplete
       next if ds.next_preprocessing(false).nil?
 
-      o = true
+      o = true if ds.ref?
       queue_job(:d, ds)
     end
     o

data/lib/miga/dataset.rb

@@ -97,7 +97,10 @@ class MiGA::Dataset < MiGA::MiGA
 
   ##
   # Inactivate a dataset. This halts automated processing by the daemon
-  def inactivate!
+  # 
+  # If given, the +reason+ string is saved as a metadata +:warn+ entry
+  def inactivate!(reason = nil)
+    metadata[:warn] = "Inactive: #{reason}" unless reason.nil?
     metadata[:inactive] = true
     metadata.save
     pull_hook :on_inactivate
@@ -107,6 +110,7 @@ class MiGA::Dataset < MiGA::MiGA
   # Activate a dataset. This removes the +:inactive+ flag
   def activate!
     metadata[:inactive] = nil
+    metadata[:warn] = nil if metadata[:warn] && metadata[:warn] =~ /^Inactive: /
     metadata.save
     pull_hook :on_activate
   end

data/lib/miga/dataset/base.rb

@@ -35,8 +35,8 @@ module MiGA::Dataset::Base
     mytaxa: '07.annotation/02.taxonomy/01.mytaxa',
     mytaxa_scan: '07.annotation/03.qa/02.mytaxa_scan',
     # Distances (for single-species datasets)
-    distances: '09.distances',
     taxonomy: '09.distances/05.taxonomy',
+    distances: '09.distances',
     # General statistics
     stats: '90.stats'
   }

data/lib/miga/dataset/hooks.rb

@@ -52,15 +52,15 @@ module MiGA::Dataset::Hooks
   end
 
   ##
-  # Run +cmd+ in the command-line with {{variables}}: dataset, project, miga,
-  # object (as defined for the event, if any)
+  # Run +cmd+ in the command-line with {{variables}}:
+  # dataset, project, project_name, miga, object (if defined for the event)
   # - +hook_args+: +[cmd]+
   # - +event_args+: +[object (optional)]+
   def hook_run_cmd(hook_args, event_args)
     Process.wait(
       spawn hook_args.first.miga_variables(
-        dataset: name, project: project.path, miga: MiGA::MiGA.root_path,
-        object: event_args.first
+        dataset: name, project: project.path, project_name: project.name,
+        miga: MiGA::MiGA.root_path, object: event_args.first
       )
     )
   end

data/lib/miga/dataset/result.rb

@@ -70,7 +70,7 @@ module MiGA::Dataset::Result
         false
       elsif add_result(t, save).nil?
         if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
-          inactivate!
+          inactivate! "Too many errors in step #{t}"
           false
         else
           true

data/lib/miga/project/hooks.rb

@@ -26,14 +26,15 @@ module MiGA::Project::Hooks
   end
 
   ##
-  # Run +cmd+ in the command-line with {{variables}}: project, miga,
-  # object (as defined by the event, if any)
+  # Run +cmd+ in the command-line with {{variables}}:
+  # project, project_name, miga, object (if defined by the event)
   # - +hook_args+: +[cmd]+
   # - +event_args+: +[object (optional)]+
   def hook_run_cmd(hook_args, event_args)
     Process.wait(
       spawn hook_args.first.miga_variables(
-        project: path, miga: MiGA::MiGA.root_path, object: event_args.first
+        project: path, project_name: name,
+        miga: MiGA::MiGA.root_path, object: event_args.first
       )
     )
   end

data/lib/miga/remote_dataset/download.rb

@@ -94,11 +94,13 @@ class MiGA::RemoteDataset
       @timeout_try = 0
       begin
         DEBUG 'GET: ' + url
-        open(url, read_timeout: 600) { |f| doc = f.read }
+        URI.parse(url).open(read_timeout: 600) { |f| doc = f.read }
       rescue => e
         @timeout_try += 1
         raise e if @timeout_try >= 3
 
+        sleep 5 # <- For: 429 Too Many Requests
+        DEBUG "RETRYING after: #{e}"
         retry
       end
       doc

data/lib/miga/result.rb

@@ -45,10 +45,6 @@ class MiGA::Result < MiGA::MiGA
   # Hash with the result metadata
   attr_reader :data
 
-  ##
-  # Array of MiGA::Result objects nested within the result (if any)
-  attr_reader :results
-
   ##
   # Load or create the MiGA::Result described by the JSON file +path+
   def initialize(path)
@@ -78,9 +74,9 @@ class MiGA::Result < MiGA::MiGA
     when :json
       @path
     when :start
-      @path.sub(/\.json$/, ".start")
+      @path.sub(/\.json$/, '.start')
     when :done
-      @path.sub(/\.json$/, ".done")
+      @path.sub(/\.json$/, '.done')
     end
   end
 
@@ -134,7 +130,7 @@ class MiGA::Result < MiGA::MiGA
   ##
   # Initialize and #save empty result
   def create
-    @data = { created: Time.now.to_s, results: [], stats: {}, files: {} }
+    @data = { created: Time.now.to_s, stats: {}, files: {} }
     save
   end
 
@@ -156,19 +152,20 @@ class MiGA::Result < MiGA::MiGA
   def load
     @data = MiGA::Json.parse(path)
     @data[:files] ||= {}
-    @results = (self[:results] || []).map { |rs| MiGA::Result.new rs }
   end
 
   ##
   # Remove result, including all associated files
   def remove!
-    each_file do |file|
-      f = File.expand_path(file, dir)
-      FileUtils.rm_rf(f)
-    end
-    %w(.start .done).each do |ext|
-      f = path.sub(/\.json$/, ext)
-      File.unlink f if File.exist? f
+    each_file { |file| FileUtils.rm_rf(File.join(dir, file)) }
+    unlink
+  end
+
+  # Unlink result by removing the .done and .start timestamps and the
+  # .json descriptor, but don't remove any other associated files
+  def unlink
+    %i(start done).each do |i|
+      f = path(i) and File.exists?(f) and File.unlink(f)
     end
     File.unlink path
   end
@@ -182,28 +179,19 @@ class MiGA::Result < MiGA::MiGA
   # Note that multiple files may have the same symbol (file_sym), since
   # arrays of files are supported.
   def each_file(&blk)
+    return to_enum(:each_file) unless block_given?
+
     @data[:files] ||= {}
     self[:files].each do |k, files|
       files = [files] unless files.kind_of? Array
       files.each do |file|
         case blk.arity
-        when 1
-          blk.call(file)
-        when 2
-          blk.call(k, file)
-        when 3
-          blk.call(k, file, File.expand_path(file, dir))
-        else
-          raise "Wrong number of arguments: #{blk.arity} for 1..3"
+        when 1; blk.call(file)
+        when 2; blk.call(k, file)
+        when 3; blk.call(k, file, File.expand_path(file, dir))
+        else; raise "Wrong number of arguments: #{blk.arity} for 1..3"
         end
       end
     end
   end
-
-  ##
-  # Add the MiGA::Result +result+ as part of the current result
-  def add_result(result)
-    @data[:results] << result.path
-    save
-  end
 end

data/lib/miga/result/stats.rb

@@ -8,6 +8,7 @@ module MiGA::Result::Stats
   # (Re-)calculate and save the statistics for the result
   def compute_stats
     method = :"compute_stats_#{key}"
+    MiGA::MiGA.DEBUG "Result(#{key}).compute_stats"
     stats = self.respond_to?(method, true) ? send(method) : nil
     unless stats.nil?
       self[:stats] = stats
@@ -109,20 +110,8 @@ module MiGA::Result::Stats
         end
       end
     else
-      # Fix estimate by domain
-      if !(tax = source.metadata[:tax]).nil? &&
-         %w[Archaea Bacteria].include?(tax[:d]) &&
-         file_path(:raw_report).nil?
-        scr = "#{MiGA::MiGA.root_path}/utils/domain-ess-genes.rb"
-        rep = file_path(:report)
-        rc_p = File.expand_path('.miga_rc', ENV['HOME'])
-        rc = File.exist?(rc_p) ? ". '#{rc_p}' && " : ''
-        $stderr.print `#{rc} ruby '#{scr}' \
-          '#{rep}' '#{rep}.domain' '#{tax[:d][0]}'`
-        add_file(:raw_report, "#{source.name}.ess/log")
-        add_file(:report, "#{source.name}.ess/log.domain")
-      end
-      # Extract/compute quality values
+      # Estimate quality metrics
+      fix_essential_genes_by_domain
       stats = { completeness: [0.0, '%'], contamination: [0.0, '%'] }
       File.open(file_path(:report), 'r') do |fh|
         fh.each_line do |ln|
@@ -131,6 +120,8 @@ module MiGA::Result::Stats
           end
         end
       end
+
+      # Determine qualitative range
       stats[:quality] = stats[:completeness][0] - stats[:contamination][0] * 5
       source.metadata[:quality] =
         case stats[:quality]
@@ -140,6 +131,12 @@ module MiGA::Result::Stats
         else; :low
         end
       source.save
+
+      # Inactivate low-quality datasets
+      min_qual = (project.metadata[:min_qual] || 50)
+      if min_qual != 'no' && stats[:quality] < min_qual
+        source.inactivate! 'Low genome quality'
+      end
     end
     stats
   end
@@ -168,12 +165,28 @@ module MiGA::Result::Stats
       stats[:aai] = [$2.to_f, '%']
       3.times { fh.gets }
       fh.each_line do |ln|
-        row = ln.chomp.gsub(/^\s*/, '').split(/\s+/)
-        break if row.empty?
+        next unless ln.chomp =~ /^\s*(\S+)\s+(.+)\s+([0-9\.e-]+)\s+\**\s*$/
 
-        stats[:"#{row[0]}_pvalue"] = row[2].to_f unless row[0] == 'root'
+        stats[:"#{$1}_pvalue"] = $3.to_f unless $1 == 'root'
       end
     end
     stats
   end
+
+  # Fix estimates based on essential genes based on taxonomy
+  def fix_essential_genes_by_domain
+    return if (tax = source.metadata[:tax]).nil? ||
+       !%w[Archaea Bacteria].include?(tax[:d]) ||
+       file_path(:raw_report)
+
+    MiGA::MiGA.DEBUG "Fixing essential genes by domain"
+    scr = "#{MiGA::MiGA.root_path}/utils/domain-ess-genes.rb"
+    rep = file_path(:report)
+    rc_p = File.expand_path('.miga_rc', ENV['HOME'])
+    rc = File.exist?(rc_p) ? ". '#{rc_p}' && " : ''
+    $stderr.print `#{rc} ruby '#{scr}' \
+      '#{rep}' '#{rep}.domain' '#{tax[:d][0]}'`
+    add_file(:raw_report, "#{source.name}.ess/log")
+    add_file(:report, "#{source.name}.ess/log.domain")
+  end
 end

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 5, 0]
+  VERSION = [0.7, 10, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 5, 13)
+  VERSION_DATE = Date.new(2020, 6, 29)
 
   ##
   # Reference of MiGA.

data/scripts/distances.bash

@@ -9,7 +9,12 @@ cd "$PROJECT/data/09.distances"
 # Initialize
 miga date > "$DATASET.start"
 
-# Run
+# Check quality first
+miga stats -P "$PROJECT" -D "$DATASET" -r essential_genes --compute-and-save
+inactive=$(miga ls -P "$PROJECT" -D "$DATASET" -m inactive | cut -f 2)
+[[ "$inactive" == "true" ]] && exit
+
+# Run distances
 ruby -I "$MIGA/lib" "$MIGA/utils/distances.rb" "$PROJECT" "$DATASET"
 
 # Finalize

data/scripts/project_stats.bash

@@ -11,6 +11,9 @@ cd "$DIR"
 # Initialize
 miga date > "miga-project.start"
 
+# Execute doctor
+miga doctor -P "$PROJECT" -v
+
 # Index taxonomy
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active
 

data/scripts/stats.bash

@@ -14,7 +14,7 @@ miga date > "$DATASET.start"
 # Calculate statistics
 for i in raw_reads trimmed_fasta assembly cds essential_genes ssu distances taxonomy ; do
   echo "# $i"
-  miga stats --compute-and-save -P "$PROJECT" -D "$DATASET" -r $i
+  miga stats --compute-and-save --ignore-empty -P "$PROJECT" -D "$DATASET" -r $i
 done
 
 # Finalize

data/test/daemon_test.rb

@@ -93,7 +93,7 @@ class DaemonTest < Test::Unit::TestCase
       0 => /-{20}\n/,
       1 => /MiGA:#{p.name} launched/,
       2 => /-{20}\n/,
-      5 => /Probing running jobs\n/
+      6 => /Probing running jobs\n/
     }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
   ensure
     begin

data/test/dataset_test.rb

@@ -185,11 +185,18 @@ class DatasetTest < Test::Unit::TestCase
     d = dataset
     assert_equal(:incomplete, d.status)
     assert_predicate(d, :active?)
-    d.inactivate!
+    d.inactivate! 'Too annoying'
     assert_equal(:inactive, d.status)
+    assert_equal('Inactive: Too annoying', d.metadata[:warn])
     assert_not_predicate(d, :active?)
     d.activate!
     assert_equal(:incomplete, d.status)
+    assert_nil(d.metadata[:warn])
     assert_predicate(d, :active?)
   end
+
+  def test_preprocessing_tasks
+    assert_include(MiGA::Dataset.PREPROCESSING_TASKS, :cds)
+    assert_include(MiGA::Dataset.PREPROCESSING_TASKS, :taxonomy)
+  end
 end

data/test/remote_dataset_test.rb

@@ -31,13 +31,15 @@ class RemoteDatasetTest < Test::Unit::TestCase
       assert_equal(MiGA::Taxonomy, tx.class, msg)
       assert_equal('Lentivirus', tx[:g], msg)
       assert_equal(
-        'ns:ncbi o:Ortervirales f:Retroviridae ' \
-          'g:Lentivirus s:Human_immunodeficiency_virus_2',
+        'ns:ncbi k:Pararnavirae p:Artverviricota c:Revtraviricetes ' \
+          'o:Ortervirales f:Retroviridae g:Lentivirus ' \
+          's:Human_immunodeficiency_virus_2',
         tx.to_s, msg
       )
       assert_equal(
-        'ns:ncbi d: k: p: c: o:Ortervirales f:Retroviridae ' \
-          'g:Lentivirus s:Human_immunodeficiency_virus_2 ssp: str: ds:',
+        'ns:ncbi d: k:Pararnavirae p:Artverviricota c:Revtraviricetes ' \
+          'o:Ortervirales f:Retroviridae g:Lentivirus ' \
+          's:Human_immunodeficiency_virus_2 ssp: str: ds:',
         tx.to_s(true), msg
       )
       assert_equal('ncbi', tx.namespace, msg)
@@ -99,7 +101,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
 
   def test_ref_type_status
     declare_remote_access
-    rd = MiGA::RemoteDataset.new('GCA_002849345', :assembly, :ncbi)
+    rd = MiGA::RemoteDataset.new('GCA_003144295.1', :assembly, :ncbi)
     assert { !rd.get_metadata[:is_type] }
     assert { rd.get_metadata[:is_ref_type] }
   end

data/utils/adapters.fa

@@ -300,3 +300,16 @@ AGATCGGAAGAGCGGTTCAGCAGGAATGCCGAG
 AGATCGGAAGAGCACACGTCTGAACTCCAGTCAC
 >TruSeq3_UniversalAdapter
 AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTA
+
+>Nextera_PE_PrefixNX/1
+AGATGTGTATAAGAGACAG
+>Nextera_PE_PrefixNX/2
+AGATGTGTATAAGAGACAG
+>Nextera_PE_Trans1
+TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG
+>Nextera_PE_Trans1_rc
+CTGTCTCTTATACACATCTGACGCTGCCGACGA
+>Nextera_PE_Trans2
+GTCTCGTGGGCTCGGAGATGTGTATAAGAGACAG
+>Nextera_PE_Trans2_rc
+CTGTCTCTTATACACATCTCCGAGCCCACGAGAC

data/utils/distance/pipeline.rb

@@ -29,6 +29,16 @@ module MiGA::DistanceRunner::Pipeline
     classify(clades, classif, metric, result_fh, val_cls)
   end
 
+  # Run distances against datasets listed in metadata's +:dist_req+
+  def distances_by_request(metric)
+    return unless dataset.metadata[:dist_req]
+
+    $stderr.puts 'Running distances by request'
+    dataset.metadata[:dist_req].each do |target|
+      ds = ref_project.dataset(target) and send(metric, ds)
+    end
+  end
+
   # Builds a tree with all visited medoids from any classification level
   def build_medoids_tree(metric)
     $stderr.puts "Building medoids tree (metric = #{metric})"
@@ -99,7 +109,7 @@ module MiGA::DistanceRunner::Pipeline
 
   # Transfer the taxonomy to the current dataset
   def transfer_taxonomy(tax)
-    $stderr.puts "Transferring taxonomy"
+    $stderr.puts 'Transferring taxonomy'
     return if tax.nil?
 
     pval = (project.metadata[:tax_pvalue] || 0.05).to_f

data/utils/distance/runner.rb

@@ -67,7 +67,7 @@ class MiGA::DistanceRunner
 
   # Launch analysis for reference datasets
   def go_ref!
-    $stderr.puts "Launching analysis for reference dataset"
+    $stderr.puts 'Launching analysis for reference dataset'
     # Initialize databases
     initialize_dbs! true
 
@@ -80,13 +80,13 @@ class MiGA::DistanceRunner
     end
 
     # Finalize
-    [:haai, :aai, :ani].each { |m| checkpoint! m if db_counts[m] > 0 }
+    %i[haai aai ani].each { |m| checkpoint! m if db_counts[m] > 0 }
   end
 
   ##
   # Launch analysis for query datasets
   def go_query!
-    $stderr.puts "Launching analysis for query dataset"
+    $stderr.puts 'Launching analysis for query dataset'
     # Check if project is ready
     tsk = ref_project.is_clade? ? [:subclades, :ani] : [:clade_finding, :aai]
     res = ref_project.result(tsk[0])
@@ -94,6 +94,7 @@ class MiGA::DistanceRunner
 
     # Initialize the databases
     initialize_dbs! false
+    distances_by_request(tsk[1])
     # Calculate the classification-informed AAI/ANI traverse
     results = File.expand_path("#{dataset.name}.#{tsk[1]}-medoids.tsv", home)
     fh = File.open(results, 'w')
@@ -111,7 +112,9 @@ class MiGA::DistanceRunner
           next unless r[1].to_i == val_cls
 
           ani = ani_after_aai(ref_project.dataset(r[0]), 80.0)
-          closest = { ds: r[0], ani: ani } unless ani.nil? or ani < closest[:ani]
+          unless ani.nil? || ani < closest[:ani]
+            closest = { ds: r[0], ani: ani }
+          end
         end
       end
     end
@@ -133,7 +136,7 @@ class MiGA::DistanceRunner
 
   # Launch analysis for taxonomy jobs
   def go_taxonomy!
-    $stderr.puts "Launching taxonomy analysis"
+    $stderr.puts 'Launching taxonomy analysis'
     return unless project.metadata[:ref_project]
 
     go_query! # <- yeah, it's actually the same, just different ref_project

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.5.0
+  version: 0.7.10.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-13 00:00:00.000000000 Z
+date: 2020-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -529,7 +529,7 @@ homepage: http://enve-omics.ce.gatech.edu/miga
 licenses:
 - Artistic-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - lib
 - README.md
@@ -550,8 +550,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: MiGA
 test_files: []