miga-base 0.7.20.1 → 0.7.23.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76b06be04db6bb162666752f374da5a3adab38e010acf8a07f20fc2e8f97e75b
-  data.tar.gz: 6c4f9b9118b21b11921b4c0a70ae99f3b1728d5737175ca5b9f23088ff985063
+  metadata.gz: 6ed8981c0dfc266aea53be66e06b77eaf277444a094ef460021c2713aef3bf73
+  data.tar.gz: d9f0bd3934dd5e3ba27cb58adfd1236a74ba6b003a06cd74b9da96e594a27e99
 SHA512:
-  metadata.gz: d90839141adeb99d0551d0228c7b5806b11a971852bf2ca8040f419af79330be47fc7abe7f1bffec35fe935b3857a1631c860d8e6aa877ba74eb1524d24f8153
-  data.tar.gz: baeee018956a1877065e2e9b5f689e1dc99af71cecc2705f951ab384b05b5760cffd7d85218068bb674541cb766a585add60c5b23a6209c15e2276b369d0f1b0
+  metadata.gz: f40242b96f36ce5528e2206aa4f366d047b5969a45d69cff5ac825297563f9adddaee957e562eddae34c96b8a0651fd2fec39a42d2cd278d66ea03ac35afabfc
+  data.tar.gz: ab01b09ab020c1bfcf5dd5be7a01c971e83e02066ad35f4b6a9f0c525351c81b5cdb0e80910cf4237b9e59a33c518995a073c1f625f049b6b722802541d11c1f

data/README.md

@@ -1,6 +1,6 @@
 [![Code Climate](https://codeclimate.com/github/bio-miga/miga/badges/gpa.svg)](https://codeclimate.com/github/bio-miga/miga)
 [![Test Coverage](https://codeclimate.com/github/bio-miga/miga/badges/coverage.svg)](https://codeclimate.com/github/bio-miga/miga/coverage)
-[![Build Status](https://travis-ci.org/bio-miga/miga.svg?branch=master)](https://travis-ci.org/bio-miga/miga)
+[![Build Status](https://github.com/bio-miga/miga/workflows/build/badge.svg)](https://github.com/bio-miga/miga/actions?query=workflow:build)
 [![Gem Version](https://badge.fury.io/rb/miga-base.svg)](https://badge.fury.io/rb/miga-base)
 [![Inch docs](http://inch-ci.org/github/bio-miga/miga.svg)](http://inch-ci.org/github/bio-miga/miga)
 

data/lib/miga/cli/action/add_result.rb

@@ -20,6 +20,6 @@ class MiGA::Cli::Action::AddResult < MiGA::Cli::Action
     obj = cli.load_project_or_dataset
     cli.say "Registering result: #{cli[:result]}"
     r = obj.add_result(cli[:result], true, force: cli[:force])
-    raise "Cannot add result, incomplete expected files." if r.nil?
+    raise 'Cannot add result, incomplete expected files' if r.nil?
   end
 end

data/lib/miga/cli/action/get_db.rb

@@ -2,9 +2,7 @@
 # @license Artistic-2.0
 
 require 'miga/cli/action'
-require 'net/ftp'
 require 'digest/md5'
-require 'open-uri'
 
 class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
   def parse_cli
@@ -12,7 +10,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
       database: :recommended,
       version: :latest,
       local: File.expand_path('.miga_db', ENV['MIGA_HOME']),
-      host: 'ftp://microbial-genomes.org/db',
+      host: MiGA::MiGA.known_hosts(:miga_db),
       pb: true,
       overwrite: true
     }
@@ -50,6 +48,14 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
   end
 
   def perform
+    # Quick check when the database is not an alias
+    dir = File.join(cli[:local], cli[:database].to_s)
+    if !cli[:overwrite] && Dir.exist?(dir)
+      cli.puts "Database exists: #{dir}"
+      return
+    end
+
+    # Remote manifest
     @ftp = remote_connection
     manif = remote_manifest(@ftp)
     cli.puts "# Host: #{manif[:host]}"
@@ -59,6 +65,8 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
     list_versions(db) and return
     ver = version_requested(db)
     check_target and return
+
+    # Download and expand
     file = download_file(@ftp, ver[:path])
     check_digest(ver, file)
     unarchive(file)
@@ -78,27 +86,14 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
 
   def remote_connection
     cli.say "Connecting to '#{cli[:host]}'"
-    uri = URI.parse(cli[:host])
-    raise 'Only FTP hosts are supported' unless uri.scheme == 'ftp'
-
-    ftp = Net::FTP.new(uri.host)
-    ftp.passive = true
-    ftp.login
-    ftp.chdir(uri.path)
-    ftp
+    MiGA::MiGA.remote_connection(cli[:host])
   end
 
   def download_file(ftp, path)
     cli.say "Downloading '#{path}'"
-    Dir.mkdir(cli[:local]) unless Dir.exist? cli[:local]
     file = File.expand_path(path, cli[:local])
-    filesize = ftp.size(path)
-    transferred = 0
-    ftp.getbinaryfile(path, file, 1024) do |data|
-      if cli[:pb]
-        transferred += data.size
-        cli.advance("#{path}:", transferred, filesize)
-      end
+    MiGA::MiGA.download_file_ftp(ftp, path, file) do |n, size|
+      cli.advance("#{path}:", n, size) if cli[:pb]
     end
     cli.print "\n" if cli[:pb]
     file
@@ -165,7 +160,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
   def check_target
     return false if cli[:overwrite]
 
-    file = File.expand_path(cli[:database].to_s, cli[:local])
+    file = File.join(cli[:local], cli[:database].to_s)
     if Dir.exist? file
       warn "The target directory already exists: #{file}"
       true
@@ -195,7 +190,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
 
   def unarchive(file)
     cli.say "Unarchiving #{file}"
-    `cd "#{cli[:local]}" && tar -zxf "#{file}"`
+    `cd "#{cli[:local]}" && tar -zxf "#{file}" && rm "#{file}"`
   end
 
   def register_database(manif, db, ver)

data/lib/miga/cli/action/init.rb

@@ -1,18 +1,24 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
 
 require 'miga/cli/action'
 require 'shellwords'
 
 class MiGA::Cli::Action::Init < MiGA::Cli::Action
   require 'miga/cli/action/init/daemon_helper'
+  require 'miga/cli/action/init/files_helper'
   include MiGA::Cli::Action::Init::DaemonHelper
+  include MiGA::Cli::Action::Init::FilesHelper
 
   def parse_cli
     cli.interactive = true
-    cli.defaults = { mytaxa: nil,
-                     config: File.expand_path('.miga_modules', ENV['HOME']),
-                     ask: false, auto: false, dtype: :bash }
+    cli.defaults = {
+      mytaxa: nil,
+      rdp: nil,
+      config: File.join(ENV['MIGA_HOME'], '.miga_modules'),
+      ask: false,
+      auto: false,
+      dtype: :bash
+    }
     cli.parse do |opt|
       opt.on(
         '-c', '--config PATH',
@@ -21,9 +27,14 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
       ) { |v| cli[:config] = v }
       opt.on(
         '--[no-]mytaxa',
-        'Should I try setting up MyTaxa its dependencies?',
+        'Should I try setting up MyTaxa and its dependencies?',
         'By default: interactive (true if --auto)'
       ) { |v| cli[:mytaxa] = v }
+      opt.on(
+        '--[no-]rdp',
+        'Should I try setting up the RDP classifier?',
+        'By default: interactive (true if --auto)'
+      ) { |v| cli[:rdp] = v }
       opt.on(
         '--daemon-type STRING',
         'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
@@ -47,13 +58,13 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     BANNER
     list_requirements
     rc_fh = open_rc_file
-    check_configuration_script rc_fh
-    paths = check_software_requirements rc_fh
-    check_additional_files paths
-    check_r_packages paths
-    check_ruby_gems paths
+    check_configuration_script(rc_fh)
+    paths = check_software_requirements(rc_fh)
+    check_additional_files(paths)
+    check_r_packages(paths)
+    check_ruby_gems(paths)
     configure_daemon
-    close_rc_file rc_fh
+    close_rc_file(rc_fh)
     cli.puts 'Configuration complete. MiGA is ready to work!'
     cli.puts ''
   end
@@ -116,58 +127,18 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
 
   private
 
-  def open_rc_file
-    rc_path = File.expand_path('.miga_rc', ENV['HOME'])
-    if File.exist? rc_path
-      if cli.ask_user(
-        'I found a previous configuration. Do you want to continue?',
-        'yes', %w(yes no)
-      ) == 'no'
-        cli.puts 'OK, see you soon!'
-        exit(0)
-      end
-    end
-    rc_fh = File.open(rc_path, 'w')
-    rc_fh.puts <<~BASH
-      #!/bin/bash
-      # `miga init` made this on #{Time.now}
-
-    BASH
-    rc_fh
-  end
-
-  def check_configuration_script(rc_fh)
-    unless File.exist? cli[:config]
-      cli[:config] = cli.ask_user(
-        'Is there a script I need to load at startup?',
-        cli[:config]
-      )
-    end
-    if File.exist? cli[:config]
-      cli[:config] = File.expand_path(cli[:config])
-      cli.puts "Found bash configuration script: #{cli[:config]}"
-      rc_fh.puts "MIGA_STARTUP='#{cli[:config]}'"
-      rc_fh.puts '. "$MIGA_STARTUP"'
-    else
-      cli[:config] = '/dev/null'
-    end
-    cli.puts ''
-  end
-
   def check_software_requirements(rc_fh)
     cli.puts 'Looking for requirements:'
-    ask_for_mytaxa
-    rc_fh.puts 'export MIGA_MYTAXA="no"' unless cli[:mytaxa]
+    ask_for_optional(:mytaxa, 'MyTaxa')
+    rc_fh.puts "export MIGA_MYTAXA='#{cli[:mytaxa] ? 'yes' : 'no'}'"
+    ask_for_optional(:rdp, 'RDP classifier')
+    rc_fh.puts "export MIGA_RDP='#{cli[:rdp] ? 'yes' : 'no'}'"
     paths = {}
     rc_fh.puts 'MIGA_PATH=""'
     req_path = File.expand_path('utils/requirements.txt', MiGA.root_path)
     File.open(req_path, 'r') do |fh|
       fh.each_line do |ln|
-        next if $. < 3
-
-        r = ln.chomp.split(/\t+/)
-        next if r[0] =~ /\(opt\)$/ && !cli[:mytaxa]
-
+        r = define_software(ln) or next
         cli.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
         path = find_software(r[1])
         paths[r[1]] = File.expand_path(r[1], path).shellescape
@@ -178,11 +149,20 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     paths
   end
 
-  def ask_for_mytaxa
-    if cli[:mytaxa].nil?
-      cli[:mytaxa] =
+  def define_software(ln)
+    r = ln.chomp.split(/\t+/)
+    return if %w[Software --------].include?(r[0])
+    return if r[0] =~ /\(mytaxa\)$/ && !cli[:mytaxa]
+    return if r[0] =~ /\(rdp\)$/ && !cli[:rdp]
+
+    r
+  end
+
+  def ask_for_optional(symbol, name)
+    if cli[symbol].nil?
+      cli[symbol] =
         cli.ask_user(
-          'Should I include MyTaxa modules?',
+          "Should I include #{name} modules?",
           'yes', %w(yes no)
         ) == 'yes'
     end
@@ -209,27 +189,6 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     path
   end
 
-  def check_additional_files(paths)
-    if cli[:mytaxa]
-      cli.puts 'Looking for MyTaxa databases:'
-      mt = File.dirname paths['MyTaxa']
-      cli.print 'Looking for scores... '
-      unless Dir.exist?(File.expand_path('db', mt))
-        cli.puts "no\nExecute 'python2 #{mt}/utils/download_db.py'"
-        exit(1)
-      end
-      cli.puts 'yes'
-      cli.print 'Looking for diamond db... '
-      unless File.exist?(File.expand_path('AllGenomes.faa.dmnd', mt))
-        cli.puts "no\nDownload " \
-          "'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" \
-          "AllGenomes.faa.dmnd' into #{mt}"
-        exit(1)
-      end
-      cli.puts ''
-    end
-  end
-
   def check_r_packages(paths)
     cli.puts 'Looking for R packages:'
     %w(ape cluster vegan).each do |pkg|
@@ -266,15 +225,4 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     end
     cli.puts ''
   end
-
-  def close_rc_file(rc_fh)
-    rc_fh.puts <<~FOOT
-
-      MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
-      MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
-      MIGA_CONFIG_DATE='#{Time.now}'
-
-    FOOT
-    rc_fh.close
-  end
 end

data/lib/miga/cli/action/init/daemon_helper.rb

@@ -1,5 +1,4 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
 
 ##
 # Helper module with daemon configuration functions for MiGA::Cli::Action::Init

data/lib/miga/cli/action/init/files_helper.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+##
+# Helper module with files configuration functions for MiGA::Cli::Action::Init
+module MiGA::Cli::Action::Init::FilesHelper
+  def open_rc_file
+    rc_path = File.expand_path('.miga_rc', ENV['HOME'])
+    if File.exist? rc_path
+      if cli.ask_user(
+        'I found a previous configuration. Do you want to continue?',
+        'yes', %w(yes no)
+      ) == 'no'
+        cli.puts 'OK, see you soon!'
+        exit(0)
+      end
+    end
+    rc_fh = File.open(rc_path, 'w')
+    rc_fh.puts <<~BASH
+      #!/bin/bash
+      # `miga init` made this on #{Time.now}
+
+    BASH
+    rc_fh
+  end
+
+  def close_rc_file(rc_fh)
+    rc_fh.puts <<~FOOT
+
+      MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
+      MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
+      MIGA_CONFIG_DATE='#{Time.now}'
+
+    FOOT
+    rc_fh.close
+  end
+
+  def check_configuration_script(rc_fh)
+    unless File.exist? cli[:config]
+      cli[:config] = cli.ask_user(
+        'Is there a script I need to load at startup?',
+        cli[:config]
+      )
+    end
+    if File.exist? cli[:config]
+      cli[:config] = File.expand_path(cli[:config])
+      cli.puts "Found bash configuration script: #{cli[:config]}"
+      rc_fh.puts "MIGA_STARTUP='#{cli[:config]}'"
+      rc_fh.puts '. "$MIGA_STARTUP"'
+    else
+      cli[:config] = '/dev/null'
+    end
+    cli.puts ''
+  end
+
+  def check_additional_files(paths)
+    if cli[:mytaxa]
+      check_mytaxa_scores(paths)
+      check_mytaxa_database(paths)
+    end
+    check_rdp_classifier if cli[:rdp]
+    check_phyla_lite
+  end
+
+  def check_mytaxa_scores(paths)
+    cli.print 'Looking for MyTaxa scores... '
+    mt = File.dirname(paths['MyTaxa'])
+    unless Dir.exist?(File.join(mt, 'db'))
+      cli.puts "no\nExecute 'python2 #{mt}/utils/download_db.py'"
+      raise 'Incomplete MyTaxa installation'
+    end
+    cli.puts 'yes'
+  end
+
+  def check_mytaxa_database(paths)
+    cli.print 'Looking for MyTaxa DB... '
+    mt = File.dirname(paths['MyTaxa'])
+    dmnd_db = 'AllGenomes.faa.dmnd'
+    miga_db = File.join(ENV['MIGA_HOME'], '.miga_db')
+    home_db = File.join(miga_db, dmnd_db)
+    mt_db = File.join(mt, 'AllGenomes.faa.dmnd')
+    if File.exist?(home_db)
+      cli.puts 'yes'
+    elsif File.exist?(mt_db)
+      cli.puts 'yes, sym-linking'
+      File.symlink(mt_db, home_db)
+    else
+      cli.puts 'no, downloading'
+      MiGA::MiGA.download_file_ftp(:miga_dist, dmnd_db, home_db) do |n, size|
+        cli.advance("#{dmnd_db}:", n, size)
+      end
+      cli.puts
+    end
+  end
+
+  def check_rdp_classifier
+    cli.print 'Looking for RDP classifier... '
+    miga_db = File.join(ENV['MIGA_HOME'], '.miga_db')
+    file = 'classifier.jar'
+    path = File.join(miga_db, file)
+    if File.size?(path)
+      cli.puts 'yes'
+    else
+      cli.puts 'no, downloading'
+      arch = 'classifier.tar.gz'
+      MiGA::MiGA.download_file_ftp(
+        :miga_dist, arch, File.join(miga_db, arch)
+      ) { |n, size| cli.advance("#{arch}:", n, size) }
+      `cd '#{miga_db}' && tar zxf '#{arch}' && rm '#{arch}'`
+      cli.puts
+    end
+  end
+
+  def check_phyla_lite
+    cli.puts 'Looking for Phyla Lite... '
+    cmd = ['get_db', '-n', 'Phyla_Lite', '--no-overwrite']
+    MiGA::Cli.new(cmd).launch(true)
+  end
+end

data/lib/miga/cli/action/next_step.rb

@@ -11,16 +11,8 @@ class MiGA::Cli::Action::NextStep < MiGA::Cli::Action
   end
 
   def perform
-    p = cli.load_project
-    n = nil
-    if cli[:dataset].nil?
-      n = p.next_distances(false)
-      n ||= p.next_inclade(false)
-    else
-      d = cli.load_dataset
-      n = d.next_preprocessing if d.is_active?
-    end
-    n ||= '?'
-    cli.puts n
+    obj = cli.load_project_or_dataset
+    n = obj.next_task
+    cli.puts(n || '?')
   end
 end

data/lib/miga/common.rb

@@ -6,6 +6,7 @@ require 'miga/json'
 require 'miga/common/base'
 require 'miga/common/path'
 require 'miga/common/format'
+require 'miga/common/net'
 require 'stringio'
 
 ##
@@ -16,6 +17,7 @@ class MiGA::MiGA
 
   extend MiGA::Common::Path
   extend MiGA::Common::Format
+  extend MiGA::Common::Net
 
   ENV['MIGA_HOME'] ||= ENV['HOME']
 
@@ -64,7 +66,7 @@ class MiGA::MiGA
     adv_n = n - @_advance_time[:n]
     unless total.nil? || @_advance_time[:last].nil? || adv_n <= 0
       if adv_n.to_f/n > 0.001
-        this_time = Time.now - @_advance_time[:last]
+        this_time = (Time.now - @_advance_time[:last]).to_f
         this_avg = this_time / adv_n
         @_advance_time[:avg] ||= this_avg
         @_advance_time[:avg] = 0.9 * @_advance_time[:avg] + 0.1 * this_avg

data/lib/miga/common/net.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'net/ftp'
+require 'open-uri'
+require 'fileutils'
+
+Net::FTP.send(:remove_const, 'FTP_PORT') # just to avoid warnings
+Net::FTP.const_set('FTP_PORT', 21)
+
+##
+# General web-access functions shared throughout MiGA.
+module MiGA::Common::Net
+  ##
+  # Returns the URL of the host +name+ (Symbol)
+  def known_hosts(name)
+    case name.to_sym
+    when :miga_online_ftp
+      'ftp://microbial-genomes.org//' # <- // to simplify chdir in connection
+    when :miga_db
+      'ftp://microbial-genomes.org/db'
+    when :miga_dist
+      'ftp://microbial-genomes.org/dist'
+    else
+      raise "Unrecognized server name: #{host}"
+    end
+  end
+
+  ##
+  # Connect to an FTP +host+ (String) or a known host name (Symbol, see
+  # +.known_hosts+)
+  def remote_connection(host)
+    host = known_hosts(host) if host.is_a?(Symbol)
+    uri = URI.parse(host)
+    raise 'Only FTP hosts are currently supported' unless uri.scheme == 'ftp'
+
+    ftp = Net::FTP.new(uri.host)
+    ftp.passive = true
+    ftp.login
+    ftp.chdir(uri.path)
+    ftp
+  end
+
+  ##
+  # Download a file via FTP using the +connection+ (returned by
+  # +.remote_connection+) with remote name +file+ into local +target+.
+  #
+  # Alternatively, +connection+ can simply be the host (String) or a recognized
+  # Symbol (see +.remote_connection+), in which case the function opens the
+  # connection automatically
+  #
+  # Reports progress to the function block with two arguments: the
+  # currently transferred size and the total file size
+  def download_file_ftp(connection, file, target)
+    # Open connection unless passed
+    close_conn = false
+    if connection.is_a?(String) || connection.is_a?(Symbol)
+      connection = remote_connection(connection)
+      close_conn = true
+    end
+
+    # Prepare download
+    FileUtils.mkdir_p(File.dirname(target))
+    filesize = connection.size(file)
+    transferred = 0
+
+    # Get in chunks of 1KiB
+    connection.getbinaryfile(file, target, 1024) do |data|
+      yield(transferred += data.size, filesize) if block_given?
+    end
+
+    # Close connection if automatically opened
+    connection.close if close_conn
+  end
+end

data/lib/miga/common/with_result.rb

@@ -1,6 +1,8 @@
 ##
 # Helper module including specific functions to handle objects that
-# have results.
+# have results. The class including this module must implement methods
+# +.RESULT_DIRS+, +#ignore_task?+, +#metadata+, +#project+,
+# and +#inactivate!+.
 module MiGA::Common::WithResult
   ##
   # Result directories as a Hash
@@ -68,4 +70,40 @@ module MiGA::Common::WithResult
   def get_result(task)
     add_result(task, false)
   end
+
+  ##
+  # Get the next task from +tasks+, saving intermediate results if +save+.
+  # If +tasks+ is +nil+ (default), it uses the entire list of tasks.
+  # Returns a Symbol.
+  def next_task(tasks = nil, save = false)
+    tasks ||= result_dirs.keys
+    tasks.find do |t|
+      if ignore_task?(t)
+        # Do not run if this step is to be ignored
+        false
+      else
+        res = add_result(t, save)
+        if res.nil?
+          # Run if the step has not been calculated,
+          # unless too many attempts were already made
+          if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
+            inactivate! "Too many errors in step #{t}"
+            false
+          else
+            true
+          end
+        else
+          # Run if the step is ready but has to be recalculated
+          res.recalculate? ? true : false
+        end
+      end
+    end
+  end
+
+  ##
+  # Mark all results for recalculation
+  def recalculate_tasks(reason = nil)
+    each_result { |_k, res| res.recalculate!(reason).save }
+  end
+
 end

data/lib/miga/dataset.rb

@@ -107,6 +107,7 @@ class MiGA::Dataset < MiGA::MiGA
     metadata[:warn] = "Inactive: #{reason}" unless reason.nil?
     metadata[:inactive] = true
     metadata.save
+    project.recalculate_tasks('Reference dataset inactivated') if ref?
     pull_hook :on_inactivate
   end
 
@@ -116,6 +117,7 @@ class MiGA::Dataset < MiGA::MiGA
     metadata[:inactive] = nil
     metadata[:warn] = nil if metadata[:warn] && metadata[:warn] =~ /^Inactive: /
     metadata.save
+    project.recalculate_tasks('Reference dataset activated') if ref?
     pull_hook :on_activate
   end
 

data/lib/miga/dataset/result.rb

@@ -77,19 +77,8 @@ module MiGA::Dataset::Result
   # Returns the key symbol of the next task that needs to be executed or nil.
   # Passes +save+ to #add_result.
   def next_preprocessing(save = false)
-    first = first_preprocessing(save) or return nil
-    @@PREPROCESSING_TASKS[@@PREPROCESSING_TASKS.index(first)..-1].find do |t|
-      if ignore_task? t
-        false
-      elsif add_result(t, save).nil?
-        if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
-          inactivate! "Too many errors in step #{t}"
-          false
-        else
-          true
-        end
-      end
-    end
+    first_preprocessing(save) if save
+    next_task(nil, save)
   end
 
   ##
@@ -301,7 +290,8 @@ module MiGA::Dataset::Result
       MiGA::Result.new("#{base}.json"), name,
       longest_ssu_gene: '.ssu.fa',
       gff: '.ssu.gff',
-      all_ssu_genes: '.ssu.all.fa'
+      all_ssu_genes: '.ssu.all.fa',
+      classification: '.rdp.tsv'
     )
     opts[:is_clean] ||= false
     r.clean! if opts[:is_clean]

data/lib/miga/project.rb

@@ -97,13 +97,28 @@ class MiGA::Project < MiGA::MiGA
 
   ##
   # Is this a clade project?
-  def is_clade?
+  def clade?
     type == :clade
   end
 
+  ##
+  # Same as active? For backward compatibility
+  alias is_clade? clade?
+
   ##
   # Is this a project for multi-organism datasets?
-  def is_multi?
+  def multi?
     @@KNOWN_TYPES[type][:multi]
   end
+
+  ##
+  # Same as multi? For backward compatibility
+  alias is_multi? multi?
+
+  ##
+  # Is this project active? Currently a dummy function, returns
+  # always true.
+  def active?
+    true
+  end
 end

data/lib/miga/project/base.rb

@@ -50,14 +50,11 @@ module MiGA::Project::Base
     07.annotation/01.function/01.essential
     07.annotation/01.function/02.ssu
     07.annotation/02.taxonomy/01.mytaxa
-    07.annotation/03.qa 07.annotation/03.qa/01.checkm
-    07.annotation/03.qa/02.mytaxa_scan
+    07.annotation/03.qa 07.annotation/03.qa/02.mytaxa_scan
     08.mapping 08.mapping/01.read-ctg 08.mapping/02.read-gene
     09.distances 09.distances/01.haai 09.distances/02.aai
     09.distances/03.ani 09.distances/04.ssu 09.distances/05.taxonomy
     10.clades 10.clades/01.find 10.clades/02.ani 10.clades/03.ogs
-    10.clades/04.phylogeny 10.clades/04.phylogeny/01.essential
-    10.clades/04.phylogeny/02.core 10.clades/05.metadata
     90.stats
   ]
 

data/lib/miga/project/dataset.rb

@@ -48,11 +48,11 @@ module MiGA::Project::Dataset
   # Add dataset identified by +name+ and return MiGA::Dataset.
   def add_dataset(name)
     unless metadata[:datasets].include? name
-      ds = MiGA::Dataset.new(self, name)
+      d = MiGA::Dataset.new(self, name)
       @metadata[:datasets] << name
       @dataset_names_hash = nil # Ensure loading even if +do_not_save+ is true
       save
-      # TODO redo_project_steps if ds.ref?
+      recalculate_tasks('New reference dataset added') if d.ref? && d.active?
       pull_hook(:on_add_dataset, name)
     end
     dataset(name)
@@ -66,6 +66,7 @@ module MiGA::Project::Dataset
 
     self.metadata[:datasets].delete(name)
     save
+    recalculate_tasks('Reference dataset unlinked') if d.ref? && d.active?
     pull_hook(:on_unlink_dataset, name)
     d
   end

data/lib/miga/project/result.rb

@@ -23,6 +23,11 @@ module MiGA::Project::Result
     self
   end
 
+  ##
+  # Do nothing, only to comply with MiGA::Common::WithResult
+  def inactivate!(reason = nil)
+  end
+
   ##
   # Is this +task+ to be bypassed?
   def ignore_task?(task)
@@ -45,17 +50,6 @@ module MiGA::Project::Result
     next_task(@@INCLADE_TASKS, save)
   end
 
-  ##
-  # Get the next task from +tasks+, saving intermediate results if +save+.
-  # If +tasks+ is +nil+ (default), it uses the entire list of tasks.
-  # Returns a Symbol.
-  def next_task(tasks = nil, save = true)
-    tasks ||= @@DISTANCE_TASKS + @@INCLADE_TASKS
-    tasks.find do |t|
-      ignore_task?(t) ? false : add_result(t, save).nil?
-    end
-  end
-
   private
 
   ##

data/lib/miga/remote_dataset.rb

@@ -162,6 +162,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
       txt.empty? ? sleep(1) : break
     end
     doc = MiGA::Json.parse(txt, symbolize: false, contents: true)
+    return if doc.nil? || doc['result'].nil? || doc['result'].empty?
+
     @_ncbi_asm_json_doc = doc['result'][ doc['result']['uids'].first ]
   end
 

data/lib/miga/result.rb

@@ -59,9 +59,25 @@ class MiGA::Result < MiGA::MiGA
   end
 
   ##
-  # Register the result as cleaned
+  # Register the result as cleaned, returns self
   def clean!
     self[:clean] = true
+    self
+  end
+
+  ##
+  # Is the result marked to be recalculated? Returns Boolean
+  def recalculate?
+    !!self[:recalculate]
+  end
+
+  ##
+  # Mark the result to be recalculated, returns self
+  def recalculate!(reason = nil)
+    self[:recalculate] = true
+    self[:recalculate_why] = reason
+    self[:recalculate_when] = Time.now.to_s
+    self
   end
 
   ##

data/lib/miga/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'date'
 
 ##
@@ -8,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 20, 1]
+  VERSION = [0.7, 23, 0].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 1, 10)
+  VERSION_DATE = Date.new(2021, 2, 9)
 
   ##
   # Reference of MiGA.

data/scripts/aai_distances.bash

@@ -19,7 +19,7 @@ rm -f miga-project.txt
   for i in $DS ; do
     echo "SELECT CASE WHEN omega!=0 THEN 'AAI' ELSE 'hAAI_AAI' END," \
       " seq1, seq2, aai, sd, n, omega from aai;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/scripts/ani_distances.bash

@@ -18,7 +18,7 @@ rm -f miga-project.txt
   echo "metric a b value sd n omega" | tr " " "\\t"
   for i in $DS ; do
     echo "SELECT 'ANI', seq1, seq2, ani, sd, n, omega from ani ;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/scripts/haai_distances.bash

@@ -22,7 +22,7 @@ rm -f miga-project.txt
   echo "metric a b value sd n omega" | tr " " "\\t"
   for i in $DS ; do
     echo "SELECT 'hAAI', seq1, seq2, aai, sd, n, omega from aai ;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/scripts/mytaxa.bash

@@ -14,15 +14,16 @@ if [[ "$MIGA_MYTAXA" == "no" ]] ; then
   echo "This system doesn't currently support MyTaxa." \
     > "$DATASET.nomytaxa.txt"
 else
-  MT=$(dirname -- "$(which MyTaxa)")
-
   # Check type of dataset
   MULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --multi \
     | wc -l | awk '{print $1}')
   if [[ "$MULTI" -eq "1" ]] ; then
     # Check requirements
-    if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
-      echo "Cannot locate the database: $MT/AllGenomes.faa.dmnd:" \
+    MT=$(dirname -- "$(which MyTaxa)")
+    DB="$MIGA_HOME/.miga_db/AllGenomes.faa.dmnd"
+    [[ -e "$DB" ]] || DB="$MT/AllGenomes.faa.dmnd"
+    if [[ ! -e "$DB" ]] ; then
+      echo "Cannot locate the database: AllGenomes.faa.dmnd:" \
             "no such file or directory" >&2
       exit 1
     fi
@@ -40,7 +41,7 @@ else
     # Execute search
     FAA="../../../06.cds/$DATASET.faa"
     [[ -s "$FAA" ]] || FAA="${FAA}.gz"
-    diamond blastp -q "$FAA" -d "$MT/AllGenomes.faa" \
+    diamond blastp -q "$FAA" -d "$DB" \
       -a "$DATASET.daa" -k 5 -p "$CORES" --min-score 60
     diamond view -a "$DATASET.daa" -o "$DATASET.blast"
 

data/scripts/mytaxa_scan.bash

@@ -5,7 +5,6 @@ SCRIPT="mytaxa_scan"
 # shellcheck source=scripts/miga.bash
 . "$MIGA/scripts/miga.bash" || exit 1
 DIR="$PROJECT/data/07.annotation/03.qa/02.mytaxa_scan"
-[[ -d "$DIR" ]] || mkdir -p "$DIR"
 cd "$DIR"
 
 # Initialize
@@ -14,17 +13,16 @@ if [[ "$MIGA_MYTAXA" == "no" ]] ; then
   echo "This system doesn't currently support MyTaxa." \
     > "$DATASET.nomytaxa.txt"
 else
-  MT=$(dirname -- "$(which MyTaxa)")
-  TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
-  trap "rm -rf '$TMPDIR'; exit" SIGHUP SIGINT SIGTERM
-
   # Check type of dataset
   NOMULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --no-multi \
     | wc -l | awk '{print $1}')
   if [[ "$NOMULTI" -eq "1" ]] ; then
     # Check requirements
+    MT=$(dirname -- "$(which MyTaxa)")
+    DB="$MIGA_HOME/.miga_db/AllGenomes.faa.dmnd"
+    [[ -e "$DB" ]] || DB="$MT/AllGenomes.faa.dmnd"
     if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
-      echo "Cannot locate the database: $MT/AllGenomes.faa.dmnd:" \
+      echo "Cannot locate the database: AllGenomes.faa.dmnd:" \
             "no such file or directory" >&2
       exit 1
     fi
@@ -39,13 +37,16 @@ else
       exit 1
     fi
      
+    TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
+    trap "rm -rf '$TMPDIR'; exit" SIGHUP SIGINT SIGTERM
+
     FAA="../../../06.cds/$DATASET.faa"
     [[ -s "$FAA" ]] || FAA="${FAA}.gz"
     if [[ ! -s "$DATASET.mytaxa" ]] ; then
       # Execute search
       if [[ ! -s "$DATASET.blast" ]] ; then
         diamond blastp -q "$FAA" -a "$DATASET.daa" -t "$TMPDIR" \
-          -d "$MT/AllGenomes.faa" -k 5 -p "$CORES" --min-score 60
+          -d "$DB" -k 5 -p "$CORES" --min-score 60
         diamond view -a "$DATASET.daa" -o "$DATASET.blast" -t "$TMPDIR"
       fi
 

data/scripts/ssu.bash

@@ -16,6 +16,7 @@ if [[ -s $fa ]] ; then
   # Run barrnap
   barrnap --quiet --threads "$CORES" "$fa" | grep "^##gff\\|;product=16S " \
     > "$DATASET.ssu.gff"
+
   # Extract
   bedtools getfasta -s "-fi" "$fa" -bed "$DATASET.ssu.gff" \
     -fo "$DATASET.ssu.all.fa"
@@ -24,9 +25,22 @@ if [[ -s $fa ]] ; then
   FastA.filter.pl "$DATASET.ssu.fa.id" "$DATASET.ssu.all.fa" > "$DATASET.ssu.fa"
   rm "$DATASET.ssu.fa.id"
   [[ -e "$fa.fai" ]] && rm "$fa.fai"
+
+  # RDP classifier
+  if [[ "$MIGA_RDP" == "yes" && -s "$DATASET.ssu.all.fa" ]] ; then
+    java -jar "$MIGA_HOME/.miga_db/classifier.jar" classify \
+      -c 0.8 -f fixrank -g 16srrna -o "$DATASET.rdp.tsv" \
+      "$DATASET.ssu.all.fa"
+    echo "# Version: $(perl -pe 's/.*://' \
+          < "$MIGA_HOME/.miga_db/classifier.version.txt" \
+          | grep . | paste - - | perl -pe 's/\t/; /')" \
+      >> "$DATASET.rdp.tsv"
+  fi
+
   # Gzip
-  gzip -9 -f "$DATASET.ssu.gff"
-  gzip -9 -f "$DATASET.ssu.all.fa"
+  for x in ssu.gff ssu.all.fa rdp.tsv ; do
+    [[ -e "${DATASET}.${x}" ]] && gzip -9 -f "${DATASET}.${x}"
+  done
 fi
 
 # Finalize

data/test/hook_test.rb

@@ -107,10 +107,10 @@ class HookTest < Test::Unit::TestCase
         File.join(project.path, 'data', '90.stats', "miga-project.#{ext}")
       )
     end
-    assert_equal(:project_stats, project.next_task(nil, false))
+    assert_equal(:project_stats, project.next_task)
     assert_equal(:test, $res)
     assert_equal(1, $counter)
-    assert_equal(:haai_distances, project.next_task)
+    assert_equal(:haai_distances, project.next_task(nil, true))
     assert_equal(:project_stats, $res)
     assert_equal(2, $counter)
   end

data/test/net_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class FormatTest < Test::Unit::TestCase
+  include TestHelper
+
+  def test_known_hosts
+    m = MiGA::MiGA
+    assert_not_nil(m.known_hosts(:miga_db))
+    assert_not_nil(m.known_hosts('miga_db'))
+    assert_not_nil(m.known_hosts(:miga_dist))
+    assert_raise { m.known_kosts(:not_a_host) }
+  end
+
+  def test_remote_connection
+    declare_remote_access
+    m = MiGA::MiGA
+    assert_raise { m.remote_connection(:bad_descriptor) }
+    assert_raise { m.remote_connection('http://microbial-genomes.org/') }
+    c = m.remote_connection(:miga_db)
+    assert_equal(Net::FTP, c.class)
+    c.close
+  end
+
+  def test_download_file_ftp
+    declare_remote_access
+    m = MiGA::MiGA
+    f = tmpfile('t/test.txt')
+    d = File.dirname(f)
+    assert(!Dir.exist?(d))
+    m.download_file_ftp(:miga_online_ftp, 'test.txt', f)
+    assert(Dir.exist?(d))
+    assert_equal('miga', File.read(f).chomp)
+  end
+end

data/utils/cleanup-databases.rb

@@ -15,7 +15,7 @@ m.say 'Cleaning Databases'
 (0..thr - 1).each do |t|
   fork do
     dsn.each_with_index do |i, idx|
-      m.advance('Dataset:', dsn.size, idx + 1) if t == 0
+      m.advance('Dataset:', idx + 1, dsn.size) if t == 0
       next unless (idx % thr) == t
 
       d = p.dataset(i)

data/utils/requirements.txt

@@ -1,10 +1,12 @@
 Software	Test exec	Website								Notes
 --------	---------	-------								-----
 Ruby		ruby		https://www.ruby-lang.org/					Required version: 2.3+
-Python		python		https://www.python.org/
+Python		python		https://www.python.org/						Required version: 3+
 R		R		http://www.r-project.org/
 SQLite3		sqlite3		https://www.sqlite.org/
 NCBI BLAST+	blastp		ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/LATEST
+DIAMOND		diamond		http://ab.inf.uni-tuebingen.de/software/diamond			Required version: v0.9.20+
+FastANI		fastANI		https://github.com/ParBLiSS/FastANI				Required version: 1.1+
 HMMer 3.0+	hmmsearch	http://hmmer.janelia.org/software
 Bedtools	bedtools	http://bedtools.readthedocs.org/en/latest/
 Prodigal	prodigal	http://prodigal.ornl.gov
@@ -14,7 +16,6 @@ Barrnap		barrnap		http://www.vicbioinformatics.com/software.barrnap.shtml
 Scythe		scythe		https://github.com/vsbuffalo/scythe				Required version: 0.991+
 FastQC		fastqc		http://www.bioinformatics.babraham.ac.uk/projects/fastqc
 SolexaQA++	SolexaQA++	http://solexaqa.sourceforge.net					Required version: v3.1.3+
-DIAMOND (opt)	diamond		http://ab.inf.uni-tuebingen.de/software/diamond			Required version: v0.9.20+
-MyTaxa (opt)	MyTaxa		http://enve-omics.ce.gatech.edu/mytaxa				The folder must contain the db and utils dirs, and the AllGenomes.faa.dmnd database
-Krona (opt)	ktImportText	https://github.com/marbl/Krona/wiki
-FastANI (opt)	fastANI		https://github.com/ParBLiSS/FastANI				Required version: 1.1+
+OpenJDK (rdp)	java		https://adoptopenjdk.net/					Any Java VM would work
+MyTaxa (mytaxa)	MyTaxa		http://enve-omics.ce.gatech.edu/mytaxa
+Krona (mytaxa)	ktImportText	https://github.com/marbl/Krona/wiki

data/utils/subclade/runner.rb

@@ -15,7 +15,7 @@ class MiGA::SubcladeRunner
     @step = step.to_sym
     @home = File.join(
       File.join(project.path, 'data', '10.clades'),
-      @step == :clade_finding ? '01.find' : '02.ani'
+      @step == :clade_finding ? '01.find.running' : '02.ani.running'
     )
     @opts[:thr] ||= ENV.fetch('CORES') { 2 }.to_i
     @opts[:run_clades] = !!@project.metadata.data.fetch(:run_clades) { true }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.20.1
+  version: 0.7.23.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-10 00:00:00.000000000 Z
+date: 2021-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -145,6 +145,7 @@ files:
 - lib/miga/cli/action/index_wf.rb
 - lib/miga/cli/action/init.rb
 - lib/miga/cli/action/init/daemon_helper.rb
+- lib/miga/cli/action/init/files_helper.rb
 - lib/miga/cli/action/lair.rb
 - lib/miga/cli/action/ln.rb
 - lib/miga/cli/action/ls.rb
@@ -169,6 +170,7 @@ files:
 - lib/miga/common/base.rb
 - lib/miga/common/format.rb
 - lib/miga/common/hooks.rb
+- lib/miga/common/net.rb
 - lib/miga/common/path.rb
 - lib/miga/common/with_daemon.rb
 - lib/miga/common/with_daemon_class.rb
@@ -235,6 +237,7 @@ files:
 - test/json_test.rb
 - test/lair_test.rb
 - test/metadata_test.rb
+- test/net_test.rb
 - test/project_test.rb
 - test/remote_dataset_test.rb
 - test/result_stats_test.rb