miga-base 0.7.22.2 → 0.7.23.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c0bdcbce7b3490a9b011ab551789b11a48169404f92ef6e61aa8e1f3a05d998
-  data.tar.gz: ab789c8da45a841561fc6b1d91717907cd54fc52934607997e895d27e5bdf636
+  metadata.gz: 6ed8981c0dfc266aea53be66e06b77eaf277444a094ef460021c2713aef3bf73
+  data.tar.gz: d9f0bd3934dd5e3ba27cb58adfd1236a74ba6b003a06cd74b9da96e594a27e99
 SHA512:
-  metadata.gz: 8c8177696eb041e7628810371b1fb347c831aaf2be5f74b0861b789520f656c8c275505b01c017802afb31439c2e7e20d940274e4493f77bf09b23e11703fadc
-  data.tar.gz: ae768cd8ff0037e109d9ab6e9a6bce448e9cdf56bc763b6e5a6571204ae33d7ffd26194fa47ae2380d028bf3fb37533f7a620c3f6cd57ac970c6bd7e432d4dcb
+  metadata.gz: f40242b96f36ce5528e2206aa4f366d047b5969a45d69cff5ac825297563f9adddaee957e562eddae34c96b8a0651fd2fec39a42d2cd278d66ea03ac35afabfc
+  data.tar.gz: ab01b09ab020c1bfcf5dd5be7a01c971e83e02066ad35f4b6a9f0c525351c81b5cdb0e80910cf4237b9e59a33c518995a073c1f625f049b6b722802541d11c1f

data/README.md

@@ -1,6 +1,6 @@
 [![Code Climate](https://codeclimate.com/github/bio-miga/miga/badges/gpa.svg)](https://codeclimate.com/github/bio-miga/miga)
 [![Test Coverage](https://codeclimate.com/github/bio-miga/miga/badges/coverage.svg)](https://codeclimate.com/github/bio-miga/miga/coverage)
-[![Build Status](https://travis-ci.org/bio-miga/miga.svg?branch=master)](https://travis-ci.org/bio-miga/miga)
+[![Build Status](https://github.com/bio-miga/miga/workflows/build/badge.svg)](https://github.com/bio-miga/miga/actions?query=workflow:build)
 [![Gem Version](https://badge.fury.io/rb/miga-base.svg)](https://badge.fury.io/rb/miga-base)
 [![Inch docs](http://inch-ci.org/github/bio-miga/miga.svg)](http://inch-ci.org/github/bio-miga/miga)
 

data/lib/miga/cli/action/get_db.rb

@@ -2,9 +2,7 @@
 # @license Artistic-2.0
 
 require 'miga/cli/action'
-require 'net/ftp'
 require 'digest/md5'
-require 'open-uri'
 
 class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
   def parse_cli
@@ -12,7 +10,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
       database: :recommended,
       version: :latest,
       local: File.expand_path('.miga_db', ENV['MIGA_HOME']),
-      host: 'ftp://microbial-genomes.org/db',
+      host: MiGA::MiGA.known_hosts(:miga_db),
       pb: true,
       overwrite: true
     }
@@ -50,6 +48,14 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
   end
 
   def perform
+    # Quick check when the database is not an alias
+    dir = File.join(cli[:local], cli[:database].to_s)
+    if !cli[:overwrite] && Dir.exist?(dir)
+      cli.puts "Database exists: #{dir}"
+      return
+    end
+
+    # Remote manifest
     @ftp = remote_connection
     manif = remote_manifest(@ftp)
     cli.puts "# Host: #{manif[:host]}"
@@ -59,6 +65,8 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
     list_versions(db) and return
     ver = version_requested(db)
     check_target and return
+
+    # Download and expand
     file = download_file(@ftp, ver[:path])
     check_digest(ver, file)
     unarchive(file)
@@ -78,27 +86,14 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
 
   def remote_connection
     cli.say "Connecting to '#{cli[:host]}'"
-    uri = URI.parse(cli[:host])
-    raise 'Only FTP hosts are supported' unless uri.scheme == 'ftp'
-
-    ftp = Net::FTP.new(uri.host)
-    ftp.passive = true
-    ftp.login
-    ftp.chdir(uri.path)
-    ftp
+    MiGA::MiGA.remote_connection(cli[:host])
   end
 
   def download_file(ftp, path)
     cli.say "Downloading '#{path}'"
-    Dir.mkdir(cli[:local]) unless Dir.exist? cli[:local]
     file = File.expand_path(path, cli[:local])
-    filesize = ftp.size(path)
-    transferred = 0
-    ftp.getbinaryfile(path, file, 1024) do |data|
-      if cli[:pb]
-        transferred += data.size
-        cli.advance("#{path}:", transferred, filesize)
-      end
+    MiGA::MiGA.download_file_ftp(ftp, path, file) do |n, size|
+      cli.advance("#{path}:", n, size) if cli[:pb]
     end
     cli.print "\n" if cli[:pb]
     file
@@ -165,7 +160,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
   def check_target
     return false if cli[:overwrite]
 
-    file = File.expand_path(cli[:database].to_s, cli[:local])
+    file = File.join(cli[:local], cli[:database].to_s)
     if Dir.exist? file
       warn "The target directory already exists: #{file}"
       true
@@ -195,7 +190,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
 
   def unarchive(file)
     cli.say "Unarchiving #{file}"
-    `cd "#{cli[:local]}" && tar -zxf "#{file}"`
+    `cd "#{cli[:local]}" && tar -zxf "#{file}" && rm "#{file}"`
   end
 
   def register_database(manif, db, ver)

data/lib/miga/cli/action/init.rb

@@ -1,18 +1,24 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
 
 require 'miga/cli/action'
 require 'shellwords'
 
 class MiGA::Cli::Action::Init < MiGA::Cli::Action
   require 'miga/cli/action/init/daemon_helper'
+  require 'miga/cli/action/init/files_helper'
   include MiGA::Cli::Action::Init::DaemonHelper
+  include MiGA::Cli::Action::Init::FilesHelper
 
   def parse_cli
     cli.interactive = true
-    cli.defaults = { mytaxa: nil,
-                     config: File.expand_path('.miga_modules', ENV['HOME']),
-                     ask: false, auto: false, dtype: :bash }
+    cli.defaults = {
+      mytaxa: nil,
+      rdp: nil,
+      config: File.join(ENV['MIGA_HOME'], '.miga_modules'),
+      ask: false,
+      auto: false,
+      dtype: :bash
+    }
     cli.parse do |opt|
       opt.on(
         '-c', '--config PATH',
@@ -21,9 +27,14 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
       ) { |v| cli[:config] = v }
       opt.on(
         '--[no-]mytaxa',
-        'Should I try setting up MyTaxa its dependencies?',
+        'Should I try setting up MyTaxa and its dependencies?',
         'By default: interactive (true if --auto)'
       ) { |v| cli[:mytaxa] = v }
+      opt.on(
+        '--[no-]rdp',
+        'Should I try setting up the RDP classifier?',
+        'By default: interactive (true if --auto)'
+      ) { |v| cli[:rdp] = v }
       opt.on(
         '--daemon-type STRING',
         'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
@@ -47,13 +58,13 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     BANNER
     list_requirements
     rc_fh = open_rc_file
-    check_configuration_script rc_fh
-    paths = check_software_requirements rc_fh
-    check_additional_files paths
-    check_r_packages paths
-    check_ruby_gems paths
+    check_configuration_script(rc_fh)
+    paths = check_software_requirements(rc_fh)
+    check_additional_files(paths)
+    check_r_packages(paths)
+    check_ruby_gems(paths)
     configure_daemon
-    close_rc_file rc_fh
+    close_rc_file(rc_fh)
     cli.puts 'Configuration complete. MiGA is ready to work!'
     cli.puts ''
   end
@@ -116,58 +127,18 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
 
   private
 
-  def open_rc_file
-    rc_path = File.expand_path('.miga_rc', ENV['HOME'])
-    if File.exist? rc_path
-      if cli.ask_user(
-        'I found a previous configuration. Do you want to continue?',
-        'yes', %w(yes no)
-      ) == 'no'
-        cli.puts 'OK, see you soon!'
-        exit(0)
-      end
-    end
-    rc_fh = File.open(rc_path, 'w')
-    rc_fh.puts <<~BASH
-      #!/bin/bash
-      # `miga init` made this on #{Time.now}
-
-    BASH
-    rc_fh
-  end
-
-  def check_configuration_script(rc_fh)
-    unless File.exist? cli[:config]
-      cli[:config] = cli.ask_user(
-        'Is there a script I need to load at startup?',
-        cli[:config]
-      )
-    end
-    if File.exist? cli[:config]
-      cli[:config] = File.expand_path(cli[:config])
-      cli.puts "Found bash configuration script: #{cli[:config]}"
-      rc_fh.puts "MIGA_STARTUP='#{cli[:config]}'"
-      rc_fh.puts '. "$MIGA_STARTUP"'
-    else
-      cli[:config] = '/dev/null'
-    end
-    cli.puts ''
-  end
-
   def check_software_requirements(rc_fh)
     cli.puts 'Looking for requirements:'
-    ask_for_mytaxa
-    rc_fh.puts 'export MIGA_MYTAXA="no"' unless cli[:mytaxa]
+    ask_for_optional(:mytaxa, 'MyTaxa')
+    rc_fh.puts "export MIGA_MYTAXA='#{cli[:mytaxa] ? 'yes' : 'no'}'"
+    ask_for_optional(:rdp, 'RDP classifier')
+    rc_fh.puts "export MIGA_RDP='#{cli[:rdp] ? 'yes' : 'no'}'"
     paths = {}
     rc_fh.puts 'MIGA_PATH=""'
     req_path = File.expand_path('utils/requirements.txt', MiGA.root_path)
     File.open(req_path, 'r') do |fh|
       fh.each_line do |ln|
-        next if $. < 3
-
-        r = ln.chomp.split(/\t+/)
-        next if r[0] =~ /\(opt\)$/ && !cli[:mytaxa]
-
+        r = define_software(ln) or next
         cli.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
         path = find_software(r[1])
         paths[r[1]] = File.expand_path(r[1], path).shellescape
@@ -178,11 +149,20 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     paths
   end
 
-  def ask_for_mytaxa
-    if cli[:mytaxa].nil?
-      cli[:mytaxa] =
+  def define_software(ln)
+    r = ln.chomp.split(/\t+/)
+    return if %w[Software --------].include?(r[0])
+    return if r[0] =~ /\(mytaxa\)$/ && !cli[:mytaxa]
+    return if r[0] =~ /\(rdp\)$/ && !cli[:rdp]
+
+    r
+  end
+
+  def ask_for_optional(symbol, name)
+    if cli[symbol].nil?
+      cli[symbol] =
         cli.ask_user(
-          'Should I include MyTaxa modules?',
+          "Should I include #{name} modules?",
           'yes', %w(yes no)
         ) == 'yes'
     end
@@ -209,27 +189,6 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     path
   end
 
-  def check_additional_files(paths)
-    if cli[:mytaxa]
-      cli.puts 'Looking for MyTaxa databases:'
-      mt = File.dirname paths['MyTaxa']
-      cli.print 'Looking for scores... '
-      unless Dir.exist?(File.expand_path('db', mt))
-        cli.puts "no\nExecute 'python2 #{mt}/utils/download_db.py'"
-        exit(1)
-      end
-      cli.puts 'yes'
-      cli.print 'Looking for diamond db... '
-      unless File.exist?(File.expand_path('AllGenomes.faa.dmnd', mt))
-        cli.puts "no\nDownload " \
-          "'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" \
-          "AllGenomes.faa.dmnd' into #{mt}"
-        exit(1)
-      end
-      cli.puts ''
-    end
-  end
-
   def check_r_packages(paths)
     cli.puts 'Looking for R packages:'
     %w(ape cluster vegan).each do |pkg|
@@ -266,15 +225,4 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     end
     cli.puts ''
   end
-
-  def close_rc_file(rc_fh)
-    rc_fh.puts <<~FOOT
-
-      MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
-      MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
-      MIGA_CONFIG_DATE='#{Time.now}'
-
-    FOOT
-    rc_fh.close
-  end
 end

data/lib/miga/cli/action/init/daemon_helper.rb

@@ -1,5 +1,4 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
 
 ##
 # Helper module with daemon configuration functions for MiGA::Cli::Action::Init

data/lib/miga/cli/action/init/files_helper.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+##
+# Helper module with files configuration functions for MiGA::Cli::Action::Init
+module MiGA::Cli::Action::Init::FilesHelper
+  def open_rc_file
+    rc_path = File.expand_path('.miga_rc', ENV['HOME'])
+    if File.exist? rc_path
+      if cli.ask_user(
+        'I found a previous configuration. Do you want to continue?',
+        'yes', %w(yes no)
+      ) == 'no'
+        cli.puts 'OK, see you soon!'
+        exit(0)
+      end
+    end
+    rc_fh = File.open(rc_path, 'w')
+    rc_fh.puts <<~BASH
+      #!/bin/bash
+      # `miga init` made this on #{Time.now}
+
+    BASH
+    rc_fh
+  end
+
+  def close_rc_file(rc_fh)
+    rc_fh.puts <<~FOOT
+
+      MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
+      MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
+      MIGA_CONFIG_DATE='#{Time.now}'
+
+    FOOT
+    rc_fh.close
+  end
+
+  def check_configuration_script(rc_fh)
+    unless File.exist? cli[:config]
+      cli[:config] = cli.ask_user(
+        'Is there a script I need to load at startup?',
+        cli[:config]
+      )
+    end
+    if File.exist? cli[:config]
+      cli[:config] = File.expand_path(cli[:config])
+      cli.puts "Found bash configuration script: #{cli[:config]}"
+      rc_fh.puts "MIGA_STARTUP='#{cli[:config]}'"
+      rc_fh.puts '. "$MIGA_STARTUP"'
+    else
+      cli[:config] = '/dev/null'
+    end
+    cli.puts ''
+  end
+
+  def check_additional_files(paths)
+    if cli[:mytaxa]
+      check_mytaxa_scores(paths)
+      check_mytaxa_database(paths)
+    end
+    check_rdp_classifier if cli[:rdp]
+    check_phyla_lite
+  end
+
+  def check_mytaxa_scores(paths)
+    cli.print 'Looking for MyTaxa scores... '
+    mt = File.dirname(paths['MyTaxa'])
+    unless Dir.exist?(File.join(mt, 'db'))
+      cli.puts "no\nExecute 'python2 #{mt}/utils/download_db.py'"
+      raise 'Incomplete MyTaxa installation'
+    end
+    cli.puts 'yes'
+  end
+
+  def check_mytaxa_database(paths)
+    cli.print 'Looking for MyTaxa DB... '
+    mt = File.dirname(paths['MyTaxa'])
+    dmnd_db = 'AllGenomes.faa.dmnd'
+    miga_db = File.join(ENV['MIGA_HOME'], '.miga_db')
+    home_db = File.join(miga_db, dmnd_db)
+    mt_db = File.join(mt, 'AllGenomes.faa.dmnd')
+    if File.exist?(home_db)
+      cli.puts 'yes'
+    elsif File.exist?(mt_db)
+      cli.puts 'yes, sym-linking'
+      File.symlink(mt_db, home_db)
+    else
+      cli.puts 'no, downloading'
+      MiGA::MiGA.download_file_ftp(:miga_dist, dmnd_db, home_db) do |n, size|
+        cli.advance("#{dmnd_db}:", n, size)
+      end
+      cli.puts
+    end
+  end
+
+  def check_rdp_classifier
+    cli.print 'Looking for RDP classifier... '
+    miga_db = File.join(ENV['MIGA_HOME'], '.miga_db')
+    file = 'classifier.jar'
+    path = File.join(miga_db, file)
+    if File.size?(path)
+      cli.puts 'yes'
+    else
+      cli.puts 'no, downloading'
+      arch = 'classifier.tar.gz'
+      MiGA::MiGA.download_file_ftp(
+        :miga_dist, arch, File.join(miga_db, arch)
+      ) { |n, size| cli.advance("#{arch}:", n, size) }
+      `cd '#{miga_db}' && tar zxf '#{arch}' && rm '#{arch}'`
+      cli.puts
+    end
+  end
+
+  def check_phyla_lite
+    cli.puts 'Looking for Phyla Lite... '
+    cmd = ['get_db', '-n', 'Phyla_Lite', '--no-overwrite']
+    MiGA::Cli.new(cmd).launch(true)
+  end
+end

data/lib/miga/common.rb

@@ -6,6 +6,7 @@ require 'miga/json'
 require 'miga/common/base'
 require 'miga/common/path'
 require 'miga/common/format'
+require 'miga/common/net'
 require 'stringio'
 
 ##
@@ -16,6 +17,7 @@ class MiGA::MiGA
 
   extend MiGA::Common::Path
   extend MiGA::Common::Format
+  extend MiGA::Common::Net
 
   ENV['MIGA_HOME'] ||= ENV['HOME']
 
@@ -64,7 +66,7 @@ class MiGA::MiGA
     adv_n = n - @_advance_time[:n]
     unless total.nil? || @_advance_time[:last].nil? || adv_n <= 0
       if adv_n.to_f/n > 0.001
-        this_time = Time.now - @_advance_time[:last]
+        this_time = (Time.now - @_advance_time[:last]).to_f
         this_avg = this_time / adv_n
         @_advance_time[:avg] ||= this_avg
         @_advance_time[:avg] = 0.9 * @_advance_time[:avg] + 0.1 * this_avg

data/lib/miga/common/net.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'net/ftp'
+require 'open-uri'
+require 'fileutils'
+
+Net::FTP.send(:remove_const, 'FTP_PORT') # just to avoid warnings
+Net::FTP.const_set('FTP_PORT', 21)
+
+##
+# General web-access functions shared throughout MiGA.
+module MiGA::Common::Net
+  ##
+  # Returns the URL of the host +name+ (Symbol)
+  def known_hosts(name)
+    case name.to_sym
+    when :miga_online_ftp
+      'ftp://microbial-genomes.org//' # <- // to simplify chdir in connection
+    when :miga_db
+      'ftp://microbial-genomes.org/db'
+    when :miga_dist
+      'ftp://microbial-genomes.org/dist'
+    else
+      raise "Unrecognized server name: #{host}"
+    end
+  end
+
+  ##
+  # Connect to an FTP +host+ (String) or a known host name (Symbol, see
+  # +.known_hosts+)
+  def remote_connection(host)
+    host = known_hosts(host) if host.is_a?(Symbol)
+    uri = URI.parse(host)
+    raise 'Only FTP hosts are currently supported' unless uri.scheme == 'ftp'
+
+    ftp = Net::FTP.new(uri.host)
+    ftp.passive = true
+    ftp.login
+    ftp.chdir(uri.path)
+    ftp
+  end
+
+  ##
+  # Download a file via FTP using the +connection+ (returned by
+  # +.remote_connection+) with remote name +file+ into local +target+.
+  #
+  # Alternatively, +connection+ can simply be the host (String) or a recognized
+  # Symbol (see +.remote_connection+), in which case the function opens the
+  # connection automatically
+  #
+  # Reports progress to the function block with two arguments: the
+  # currently transferred size and the total file size
+  def download_file_ftp(connection, file, target)
+    # Open connection unless passed
+    close_conn = false
+    if connection.is_a?(String) || connection.is_a?(Symbol)
+      connection = remote_connection(connection)
+      close_conn = true
+    end
+
+    # Prepare download
+    FileUtils.mkdir_p(File.dirname(target))
+    filesize = connection.size(file)
+    transferred = 0
+
+    # Get in chunks of 1KiB
+    connection.getbinaryfile(file, target, 1024) do |data|
+      yield(transferred += data.size, filesize) if block_given?
+    end
+
+    # Close connection if automatically opened
+    connection.close if close_conn
+  end
+end

data/lib/miga/common/with_result.rb

@@ -103,7 +103,7 @@ module MiGA::Common::WithResult
   ##
   # Mark all results for recalculation
   def recalculate_tasks(reason = nil)
-    each_result { |res| res.recalculate!(reason).save }
+    each_result { |_k, res| res.recalculate!(reason).save }
   end
 
 end

data/lib/miga/dataset/result.rb

@@ -290,7 +290,8 @@ module MiGA::Dataset::Result
       MiGA::Result.new("#{base}.json"), name,
       longest_ssu_gene: '.ssu.fa',
       gff: '.ssu.gff',
-      all_ssu_genes: '.ssu.all.fa'
+      all_ssu_genes: '.ssu.all.fa',
+      classification: '.rdp.tsv'
     )
     opts[:is_clean] ||= false
     r.clean! if opts[:is_clean]

data/lib/miga/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'date'
 
 ##
@@ -8,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 22, 2]
+  VERSION = [0.7, 23, 0].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 1, 30)
+  VERSION_DATE = Date.new(2021, 2, 9)
 
   ##
   # Reference of MiGA.

data/scripts/mytaxa.bash

@@ -14,15 +14,16 @@ if [[ "$MIGA_MYTAXA" == "no" ]] ; then
   echo "This system doesn't currently support MyTaxa." \
     > "$DATASET.nomytaxa.txt"
 else
-  MT=$(dirname -- "$(which MyTaxa)")
-
   # Check type of dataset
   MULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --multi \
     | wc -l | awk '{print $1}')
   if [[ "$MULTI" -eq "1" ]] ; then
     # Check requirements
-    if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
-      echo "Cannot locate the database: $MT/AllGenomes.faa.dmnd:" \
+    MT=$(dirname -- "$(which MyTaxa)")
+    DB="$MIGA_HOME/.miga_db/AllGenomes.faa.dmnd"
+    [[ -e "$DB" ]] || DB="$MT/AllGenomes.faa.dmnd"
+    if [[ ! -e "$DB" ]] ; then
+      echo "Cannot locate the database: AllGenomes.faa.dmnd:" \
             "no such file or directory" >&2
       exit 1
     fi
@@ -40,7 +41,7 @@ else
     # Execute search
     FAA="../../../06.cds/$DATASET.faa"
     [[ -s "$FAA" ]] || FAA="${FAA}.gz"
-    diamond blastp -q "$FAA" -d "$MT/AllGenomes.faa" \
+    diamond blastp -q "$FAA" -d "$DB" \
       -a "$DATASET.daa" -k 5 -p "$CORES" --min-score 60
     diamond view -a "$DATASET.daa" -o "$DATASET.blast"
 

data/scripts/mytaxa_scan.bash

@@ -5,7 +5,6 @@ SCRIPT="mytaxa_scan"
 # shellcheck source=scripts/miga.bash
 . "$MIGA/scripts/miga.bash" || exit 1
 DIR="$PROJECT/data/07.annotation/03.qa/02.mytaxa_scan"
-[[ -d "$DIR" ]] || mkdir -p "$DIR"
 cd "$DIR"
 
 # Initialize
@@ -14,17 +13,16 @@ if [[ "$MIGA_MYTAXA" == "no" ]] ; then
   echo "This system doesn't currently support MyTaxa." \
     > "$DATASET.nomytaxa.txt"
 else
-  MT=$(dirname -- "$(which MyTaxa)")
-  TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
-  trap "rm -rf '$TMPDIR'; exit" SIGHUP SIGINT SIGTERM
-
   # Check type of dataset
   NOMULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --no-multi \
     | wc -l | awk '{print $1}')
   if [[ "$NOMULTI" -eq "1" ]] ; then
     # Check requirements
+    MT=$(dirname -- "$(which MyTaxa)")
+    DB="$MIGA_HOME/.miga_db/AllGenomes.faa.dmnd"
+    [[ -e "$DB" ]] || DB="$MT/AllGenomes.faa.dmnd"
     if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
-      echo "Cannot locate the database: $MT/AllGenomes.faa.dmnd:" \
+      echo "Cannot locate the database: AllGenomes.faa.dmnd:" \
             "no such file or directory" >&2
       exit 1
     fi
@@ -39,13 +37,16 @@ else
       exit 1
     fi
      
+    TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
+    trap "rm -rf '$TMPDIR'; exit" SIGHUP SIGINT SIGTERM
+
     FAA="../../../06.cds/$DATASET.faa"
     [[ -s "$FAA" ]] || FAA="${FAA}.gz"
     if [[ ! -s "$DATASET.mytaxa" ]] ; then
       # Execute search
       if [[ ! -s "$DATASET.blast" ]] ; then
         diamond blastp -q "$FAA" -a "$DATASET.daa" -t "$TMPDIR" \
-          -d "$MT/AllGenomes.faa" -k 5 -p "$CORES" --min-score 60
+          -d "$DB" -k 5 -p "$CORES" --min-score 60
         diamond view -a "$DATASET.daa" -o "$DATASET.blast" -t "$TMPDIR"
       fi
 

data/scripts/ssu.bash

@@ -16,6 +16,7 @@ if [[ -s $fa ]] ; then
   # Run barrnap
   barrnap --quiet --threads "$CORES" "$fa" | grep "^##gff\\|;product=16S " \
     > "$DATASET.ssu.gff"
+
   # Extract
   bedtools getfasta -s "-fi" "$fa" -bed "$DATASET.ssu.gff" \
     -fo "$DATASET.ssu.all.fa"
@@ -24,9 +25,22 @@ if [[ -s $fa ]] ; then
   FastA.filter.pl "$DATASET.ssu.fa.id" "$DATASET.ssu.all.fa" > "$DATASET.ssu.fa"
   rm "$DATASET.ssu.fa.id"
   [[ -e "$fa.fai" ]] && rm "$fa.fai"
+
+  # RDP classifier
+  if [[ "$MIGA_RDP" == "yes" && -s "$DATASET.ssu.all.fa" ]] ; then
+    java -jar "$MIGA_HOME/.miga_db/classifier.jar" classify \
+      -c 0.8 -f fixrank -g 16srrna -o "$DATASET.rdp.tsv" \
+      "$DATASET.ssu.all.fa"
+    echo "# Version: $(perl -pe 's/.*://' \
+          < "$MIGA_HOME/.miga_db/classifier.version.txt" \
+          | grep . | paste - - | perl -pe 's/\t/; /')" \
+      >> "$DATASET.rdp.tsv"
+  fi
+
   # Gzip
-  gzip -9 -f "$DATASET.ssu.gff"
-  gzip -9 -f "$DATASET.ssu.all.fa"
+  for x in ssu.gff ssu.all.fa rdp.tsv ; do
+    [[ -e "${DATASET}.${x}" ]] && gzip -9 -f "${DATASET}.${x}"
+  done
 fi
 
 # Finalize

data/test/net_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class FormatTest < Test::Unit::TestCase
+  include TestHelper
+
+  def test_known_hosts
+    m = MiGA::MiGA
+    assert_not_nil(m.known_hosts(:miga_db))
+    assert_not_nil(m.known_hosts('miga_db'))
+    assert_not_nil(m.known_hosts(:miga_dist))
+    assert_raise { m.known_kosts(:not_a_host) }
+  end
+
+  def test_remote_connection
+    declare_remote_access
+    m = MiGA::MiGA
+    assert_raise { m.remote_connection(:bad_descriptor) }
+    assert_raise { m.remote_connection('http://microbial-genomes.org/') }
+    c = m.remote_connection(:miga_db)
+    assert_equal(Net::FTP, c.class)
+    c.close
+  end
+
+  def test_download_file_ftp
+    declare_remote_access
+    m = MiGA::MiGA
+    f = tmpfile('t/test.txt')
+    d = File.dirname(f)
+    assert(!Dir.exist?(d))
+    m.download_file_ftp(:miga_online_ftp, 'test.txt', f)
+    assert(Dir.exist?(d))
+    assert_equal('miga', File.read(f).chomp)
+  end
+end

data/utils/requirements.txt

@@ -1,10 +1,12 @@
 Software	Test exec	Website								Notes
 --------	---------	-------								-----
 Ruby		ruby		https://www.ruby-lang.org/					Required version: 2.3+
-Python		python		https://www.python.org/
+Python		python		https://www.python.org/						Required version: 3+
 R		R		http://www.r-project.org/
 SQLite3		sqlite3		https://www.sqlite.org/
 NCBI BLAST+	blastp		ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/LATEST
+DIAMOND		diamond		http://ab.inf.uni-tuebingen.de/software/diamond			Required version: v0.9.20+
+FastANI		fastANI		https://github.com/ParBLiSS/FastANI				Required version: 1.1+
 HMMer 3.0+	hmmsearch	http://hmmer.janelia.org/software
 Bedtools	bedtools	http://bedtools.readthedocs.org/en/latest/
 Prodigal	prodigal	http://prodigal.ornl.gov
@@ -14,7 +16,6 @@ Barrnap		barrnap		http://www.vicbioinformatics.com/software.barrnap.shtml
 Scythe		scythe		https://github.com/vsbuffalo/scythe				Required version: 0.991+
 FastQC		fastqc		http://www.bioinformatics.babraham.ac.uk/projects/fastqc
 SolexaQA++	SolexaQA++	http://solexaqa.sourceforge.net					Required version: v3.1.3+
-DIAMOND (opt)	diamond		http://ab.inf.uni-tuebingen.de/software/diamond			Required version: v0.9.20+
-MyTaxa (opt)	MyTaxa		http://enve-omics.ce.gatech.edu/mytaxa				The folder must contain the db and utils dirs, and the AllGenomes.faa.dmnd database
-Krona (opt)	ktImportText	https://github.com/marbl/Krona/wiki
-FastANI (opt)	fastANI		https://github.com/ParBLiSS/FastANI				Required version: 1.1+
+OpenJDK (rdp)	java		https://adoptopenjdk.net/					Any Java VM would work
+MyTaxa (mytaxa)	MyTaxa		http://enve-omics.ce.gatech.edu/mytaxa
+Krona (mytaxa)	ktImportText	https://github.com/marbl/Krona/wiki

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.22.2
+  version: 0.7.23.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-30 00:00:00.000000000 Z
+date: 2021-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -145,6 +145,7 @@ files:
 - lib/miga/cli/action/index_wf.rb
 - lib/miga/cli/action/init.rb
 - lib/miga/cli/action/init/daemon_helper.rb
+- lib/miga/cli/action/init/files_helper.rb
 - lib/miga/cli/action/lair.rb
 - lib/miga/cli/action/ln.rb
 - lib/miga/cli/action/ls.rb
@@ -169,6 +170,7 @@ files:
 - lib/miga/common/base.rb
 - lib/miga/common/format.rb
 - lib/miga/common/hooks.rb
+- lib/miga/common/net.rb
 - lib/miga/common/path.rb
 - lib/miga/common/with_daemon.rb
 - lib/miga/common/with_daemon_class.rb
@@ -235,6 +237,7 @@ files:
 - test/json_test.rb
 - test/lair_test.rb
 - test/metadata_test.rb
+- test/net_test.rb
 - test/project_test.rb
 - test/remote_dataset_test.rb
 - test/result_stats_test.rb