miga-base 0.7.4.0 → 0.7.9.0

data/lib/miga/cli/action/doctor/base.rb

@@ -1,4 +1,3 @@
-
 require 'miga/cli/action'
 require 'sqlite3'
 
@@ -8,13 +7,34 @@ end
 module MiGA::Cli::Action::Doctor::Base
   ##
   # Check the database in +db_file+ maintains integrity for the
-  # tables saving +metric+ (:ani or :aai)
-  def check_sqlite3_database(db_file, metric)
+  # tables saving +metric+ (:ani or :aai) and call +blk+ if the
+  # file is corrupt or doesn't contain the expected structure
+  def check_sqlite3_database(db_file, metric, &blk)
     SQLite3::Database.new(db_file) do |conn|
       conn.execute("select count(*) from #{metric}").first
     end
-  rescue SQLite3::SQLException
-    yield
+  rescue SQLite3::SQLException, SQLite3::CorruptException
+    blk.call
+  end
+
+  def each_database_file(dataset, &blk)
+    ref_db = { '01.haai' => :aai, '02.aai' => :aai, '03.ani' => :ani }
+    qry_db = { '.haai.db' => :aai, '.aai.db' => :aai, '.ani.db' => :ani }
+    base = File.join(dataset.project.path, 'data', '09.distances')
+    result = :distances
+    if dataset.ref?
+      file_db = "#{dataset.name}.db"
+      ref_db.each do |dir, metric|
+        file = File.join(base, dir, file_db)
+        blk[file, metric, result] if File.exist? file
+      end
+      base = File.join(base, '05.taxonomy')
+      result = :taxonomy
+    end
+    qry_db.each do |ext, metric|
+      file = File.join(base, "#{dataset.name}#{ext}")
+      blk[file, metric, result] if File.exist? file
+    end
   end
 
   ##
@@ -33,8 +53,10 @@ module MiGA::Cli::Action::Doctor::Base
       lineno = 0
       fh.each_line do |ln|
         next if (lineno += 1) == 1
+
         r = ln.split("\t")
         next unless [1, 2].map { |i| p.dataset(r[i]).nil? }.any?
+
         [1, 2].each do |i|
           if p.dataset(r[i]).nil? || !p.dataset(r[i]).active?
             notok[r[i]] = true
@@ -53,6 +75,7 @@ module MiGA::Cli::Action::Doctor::Base
   # This is a subtask of +check_dist+
   def check_dist_fix(cli, p, fix)
     return if fix.empty?
+
     cli.say("- Fixing #{fix.size} datasets")
     fix.each do |d_n|
       cli.say "  > Fixing #{d_n}."
@@ -66,6 +89,7 @@ module MiGA::Cli::Action::Doctor::Base
   # This is a subtask of +check_dist+
   def check_dist_recompute(cli, res, notok)
     return if notok.empty?
+
     cli.say '- Unregistered datasets detected: '
     if notok.size <= 5
       notok.each { |i| cli.say "  > #{i}" }
@@ -76,4 +100,3 @@ module MiGA::Cli::Action::Doctor::Base
     res.remove!
   end
 end
-

data/lib/miga/cli/action/edit.rb

@@ -4,7 +4,6 @@
 require 'miga/cli/action'
 
 class MiGA::Cli::Action::Edit < MiGA::Cli::Action
-
   def parse_cli
     cli.parse do |opt|
       cli.opt_object(opt, [:project, :dataset_opt])
@@ -28,7 +27,7 @@ class MiGA::Cli::Action::Edit < MiGA::Cli::Action
     obj = cli.load_project_or_dataset
     unless cli[:activate].nil?
       cli.ensure_par({ dataset: '-D' },
-        '%<name>s is mandatory with --[in-]activate: please provide %<flag>s')
+                     '%<name>s is mandatory with --[in-]activate: please provide %<flag>s')
       cli[:activate] ? obj.activate! : obj.inactivate!
     end
     cli.add_metadata(obj)

data/lib/miga/cli/action/files.rb

@@ -4,27 +4,27 @@
 require 'miga/cli/action'
 
 class MiGA::Cli::Action::Files < MiGA::Cli::Action
-
   def parse_cli
-    cli.defaults = {details: false, json: true}
+    cli.defaults = { details: false, json: true }
     cli.parse do |opt|
       cli.opt_object(opt, [:project, :dataset_opt])
       opt.on(
         '-i', '--info',
         'Print additional details for each file'
-        ){ |v| cli[:details] = v }
-      opt.on('--[no-]json',
+      ) { |v| cli[:details] = v }
+      opt.on(
+        '--[no-]json',
         'Include (or not) JSON files containing results metadata',
         'JSON files are included by default'
-        ){ |v| cli[:json] = v }
+      ) { |v| cli[:json] = v }
     end
   end
 
   def perform
     cli.load_project_or_dataset.each_result do |sym, res|
-      cli.puts "#{ "#{sym}\tjson\t" if cli[:details] }#{res.path}" if cli[:json]
-      res.each_file do |k,f|
-        cli.puts "#{ "#{sym}\t#{k}\t" if cli[:details] }#{res.dir}/#{f}"
+      cli.puts "#{"#{sym}\tjson\t" if cli[:details]}#{res.path}" if cli[:json]
+      res.each_file do |k, f|
+        cli.puts "#{"#{sym}\t#{k}\t" if cli[:details]}#{res.dir}/#{f}"
       end
     end
   end

data/lib/miga/cli/action/find.rb

@@ -4,29 +4,28 @@
 require 'miga/cli/action'
 
 class MiGA::Cli::Action::Find < MiGA::Cli::Action
-
   def parse_cli
-    cli.defaults = {add: false, ref: true}
+    cli.defaults = { add: false, ref: true }
     cli.parse do |opt|
       cli.opt_object(opt, [:project, :dataset_type])
       opt.on(
         '-a', '--add',
         'Register the datasets found',
         'By default, only lists them (dry run)'
-        ){ |v| cli[:add] = v }
+      ) { |v| cli[:add] = v }
       opt.on(
         '-q', '--query',
         'Register datasets as query'
-        ){ |v| cli[:ref] = !v }
+      ) { |v| cli[:ref] = !v }
       opt.on(
         '-u', '--user STRING',
         'Owner of the dataset.'
-        ){ |v| cli[:user] = v }
+      ) { |v| cli[:user] = v }
       opt.on(
         '-m', '--metadata STRING',
         'Metadata as key-value pairs separated by = and delimited by comma',
         'Values are saved as strings except for booleans (true / false) or nil'
-        ){ |v| cli[:metadata] = v }
+      ) { |v| cli[:metadata] = v }
     end
   end
 

data/lib/miga/cli/action/generic.rb

@@ -4,31 +4,31 @@
 require 'miga/cli/action'
 
 class MiGA::Cli::Action::Generic < MiGA::Cli::Action
-
   def parse_cli
     cli.opt_common = false
     cli.parse do |opt|
-      descriptions = cli.class.TASK_DESC.keep_if { |k,v| k != :generic }
+      descriptions = cli.class.TASK_DESC.keep_if { |k, v| k != :generic }
       opt.separator MiGA::MiGA.tabulate(
-        [:action, :description], descriptions).join("\n")
+        [:action, :description], descriptions
+      ).join("\n")
       opt.separator ''
       opt.separator 'generic options:'
       opt.on(
         '-h', '--help',
         'Display this screen'
-        ){ puts opt ; exit }
+      ) { puts opt; exit }
       opt.on(
         '-v', '--version',
         'Show MiGA version'
-        ){ puts MiGA::MiGA.VERSION ; exit }
+      ) { puts MiGA::MiGA.VERSION; exit }
       opt.on(
         '-V', '--long-version',
         'Show complete MiGA version'
-        ){ |v| puts MiGA::MiGA.LONG_VERSION ; exit }
+      ) { |v| puts MiGA::MiGA.LONG_VERSION; exit }
       opt.on(
         '-C', '--citation',
         'How to cite MiGA'
-        ) {|v| puts MiGA::MiGA.CITATION ; exit }
+      ) { |v| puts MiGA::MiGA.CITATION; exit }
     end
   end
 

data/lib/miga/cli/action/get.rb

@@ -5,24 +5,23 @@ require 'miga/cli/action'
 require 'miga/remote_dataset'
 
 class MiGA::Cli::Action::Get < MiGA::Cli::Action
-
   def parse_cli
-    cli.defaults = {query: false, universe: :ncbi, db: :nuccore,
-      get_md: false, only_md: false}
+    cli.defaults = { query: false, universe: :ncbi, db: :nuccore,
+                     get_md: false, only_md: false }
     cli.parse do |opt|
       cli.opt_object(opt, [:project, :dataset, :dataset_type])
       opt.on(
         '-I', '--ids ID1,ID2,...', Array,
         '(Mandatory unless -F) IDs in the remote database separated by commas'
-        ){ |v| cli[:ids] = v }
+      ) { |v| cli[:ids] = v }
       opt.on(
         '-U', '--universe STRING',
         "Universe of the remote database. By default: #{cli[:universe]}"
-        ){ |v| cli[:universe] = v.to_sym }
+      ) { |v| cli[:universe] = v.to_sym }
       opt.on(
         '--db STRING',
         "Name of the remote database. By default: #{cli[:db]}"
-        ){ |v| cli[:db] = v.to_sym }
+      ) { |v| cli[:db] = v.to_sym }
       opt.on(
         '-F', '--file PATH',
         'Tab-delimited file (with header) listing the datasets to download',
@@ -30,39 +29,40 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
         'including: dataset, ids, universe, db, metadata',
         'For flags without value (like query) use true/false',
         'Unsupported values are: project, file, verbose, help, and debug'
-        ){ |v| cli[:file] = v }
+      ) { |v| cli[:file] = v }
       opt.on(
         '-q', '--query',
         'Register the dataset as a query, not a reference dataset'
-        ){ |v| cli[:query] = v }
-      opt.on('--ignore-dup',
+      ) { |v| cli[:query] = v }
+      opt.on(
+        '--ignore-dup',
         'Ignore datasets that already exist'
-        ){ |v| cli[:ignore_dup] = v }
+      ) { |v| cli[:ignore_dup] = v }
       opt.on(
         '-d', '--description STRING',
         'Description of the dataset'
-        ){ |v| cli[:description] = v }
+      ) { |v| cli[:description] = v }
       opt.on(
         '-c', '--comments STRING',
         'Comments on the dataset'
-        ){ |v| cli[:comments] = v }
+      ) { |v| cli[:comments] = v }
       opt.on(
         '-m', '--metadata STRING',
         'Metadata as key-value pairs separated by = and delimited by comma',
         'Values are saved as strings except for booleans (true / false) or nil'
-        ){ |v| cli[:metadata] = v }
+      ) { |v| cli[:metadata] = v }
       opt.on(
         '--get-metadata',
         'Only download and update metadata for existing datasets'
-        ){ |v| cli[:get_md] = v }
+      ) { |v| cli[:get_md] = v }
       opt.on(
         '--only-metadata',
         'Create datasets without input data but retrieve all metadata'
-        ){ |v| cli[:only_md] = v }
+      ) { |v| cli[:only_md] = v }
       opt.on(
         '--api-key STRING',
         'API key for the given universe'
-        ){ |v| cli[:api_key] = v }
+      ) { |v| cli[:api_key] = v }
     end
   end
 
@@ -72,6 +72,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob.each do |sub_cli|
       rd = create_remote_dataset(sub_cli)
       next if rd.nil?
+
       if sub_cli[:get_md]
         update_metadata(sub_cli, p, rd)
       else
@@ -84,13 +85,14 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
 
   def get_sub_cli
     return [cli] if cli[:file].nil?
+
     glob = []
     File.open(cli[:file], 'r') do |fh|
       h = nil
       fh.each do |ln|
         r = ln.chomp.split(/\t/)
         if h.nil?
-           h = r
+          h = r
         else
           argv_i = [self.name]
           h.each_with_index do |field, k|
@@ -132,6 +134,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     sub_cli.say 'Updating dataset'
     d = p.dataset(sub_cli[:dataset])
     return if d.nil?
+
     md = sub_cli.add_metadata(d).metadata.data
     rd.update_metadata(d, md)
   end

data/lib/miga/cli/action/get_db.rb

@@ -7,7 +7,6 @@ require 'digest/md5'
 require 'open-uri'
 
 class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
-
   def parse_cli
     cli.defaults = {
       database: :recommended,
@@ -81,6 +80,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
     cli.say "Connecting to '#{cli[:host]}'"
     uri = URI.parse(cli[:host])
     raise 'Only FTP hosts are supported' unless uri.scheme == 'ftp'
+
     ftp = Net::FTP.new(uri.host)
     ftp.passive = true
     ftp.login
@@ -113,11 +113,13 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
     [:recommended, :test].each do |n|
       if cli[:database] == n
         raise "This host has no #{n} database" if manif[n].nil?
+
         cli[:database] = manif[n].to_sym
       end
     end
     db = manif[:databases][cli[:database]]
     raise 'Cannot find database in this host' if db.nil?
+
     db
   end
 
@@ -127,12 +129,14 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
     end
     ver = db[:versions][cli[:version]]
     raise 'Cannot find database version' if ver.nil?
+
     cli.puts "# Database size: #{version_size(ver)}"
     ver
   end
 
   def list_databases(manif)
     return false unless cli[:list_databases]
+
     cli.puts "# Recommended database: #{manif[:recommended]}"
     cli.puts ''
     cli.table(
@@ -146,6 +150,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
 
   def list_versions(db)
     return false unless cli[:list_versions]
+
     cli.puts "# Database: #{cli[:database]}"
     cli.puts ''
     cli.table(
@@ -159,6 +164,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
 
   def check_target
     return false if cli[:overwrite]
+
     file = File.expand_path(cli[:database], cli[:local])
     if Dir.exist? file
       warn "The target directory already exists: #{file}"
@@ -201,7 +207,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
     reg[:databases][cli[:database]] ||= {}
     reg[:databases][cli[:database]][:manif_last_update] = manif[:last_update]
     reg[:databases][cli[:database]][:manif_host] = manif[:host]
-    db.each { |k,v| reg[:databases][cli[:database]][k] = v }
+    db.each { |k, v| reg[:databases][cli[:database]][k] = v }
     reg[:databases][cli[:database]][:local_version] = ver
     MiGA::Json.generate(reg, local_manif)
   end

data/lib/miga/cli/action/index_wf.rb

@@ -17,7 +17,7 @@ class MiGA::Cli::Action::IndexWf < MiGA::Cli::Action
       ) { |v| cli[:mytaxa] = v }
       opts_for_wf_distances(opt)
       opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)',
-        cleanup: false, project_type: true)
+                  cleanup: false, project_type: true)
     end
   end
 

data/lib/miga/cli/action/init.rb

@@ -7,44 +7,44 @@ require 'shellwords'
 class MiGA::Cli::Action::Init < MiGA::Cli::Action
   require 'miga/cli/action/init/daemon_helper'
   include MiGA::Cli::Action::Init::DaemonHelper
-  
+
   def parse_cli
     cli.interactive = true
     cli.defaults = { mytaxa: nil,
-      config: File.expand_path('.miga_modules', ENV['HOME']),
-      ask: false, auto: false, dtype: :bash }
+                     config: File.expand_path('.miga_modules', ENV['HOME']),
+                     ask: false, auto: false, dtype: :bash }
     cli.parse do |opt|
       opt.on(
         '-c', '--config PATH',
         'Path to the Bash configuration file',
         "By default: #{cli[:config]}"
-      ){ |v| cli[:config] = v }
+      ) { |v| cli[:config] = v }
       opt.on(
         '--[no-]mytaxa',
         'Should I try setting up MyTaxa its dependencies?',
         'By default: interactive (true if --auto)'
-      ){ |v| cli[:mytaxa] = v }
+      ) { |v| cli[:mytaxa] = v }
       opt.on(
         '--daemon-type STRING',
         'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
         "By default: interactive (#{cli[:dtype]} if --auto)"
-      ){ |v| cli[:dtype] = v.to_sym }
+      ) { |v| cli[:dtype] = v.to_sym }
       opt.on(
         '--ask-all',
         'Ask for the location of all software',
         'By default, only the locations missing in PATH are requested'
-      ){ |v| cli[:ask] = v }
+      ) { |v| cli[:ask] = v }
     end
   end
 
   def perform
-    cli.puts <<BANNER
-===[ Welcome to MiGA, the Microbial Genome Atlas ]===
+    cli.puts <<~BANNER
+      ===[ Welcome to MiGA, the Microbial Genome Atlas ]===
 
-I'm the initialization script, and I'll sniff around your computer to
-make sure you have all the requirements for MiGA data processing.
+      I'm the initialization script, and I'll sniff around your computer to
+      make sure you have all the requirements for MiGA data processing.
 
-BANNER
+    BANNER
     list_requirements
     rc_fh = open_rc_file
     check_configuration_script rc_fh
@@ -66,8 +66,10 @@ BANNER
   end
 
   def run_r_cmd(cli, paths, cmd)
-    run_cmd(cli,
-      "echo #{cmd.shellescape} | #{paths['R'].shellescape} --vanilla -q 2>&1")
+    run_cmd(
+      cli,
+      "echo #{cmd.shellescape} | #{paths['R'].shellescape} --vanilla -q 2>&1"
+    )
   end
 
   def test_r_package(cli, paths, pkg)
@@ -81,24 +83,30 @@ BANNER
   end
 
   def test_ruby_gem(cli, paths, pkg)
-    run_cmd(cli,
-      "#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null")
+    run_cmd(
+      cli,
+      "#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null"
+    )
     $?.success?
   end
 
   def install_ruby_gem(cli, paths, pkg)
     gem_cmd = "Gem::GemRunner.new.run %w(install --user #{pkg})"
-    run_cmd(cli, "#{paths['ruby'].shellescape} \
+    run_cmd(
+      cli,
+      "#{paths['ruby'].shellescape} \
         -r rubygems -r rubygems/gem_runner \
-        -e #{gem_cmd.shellescape} 2>&1")
+        -e #{gem_cmd.shellescape} 2>&1"
+    )
   end
 
   def list_requirements
     if cli.ask_user(
-          'Would you like to see all the requirements before starting?',
-          'no', %w(yes no)) == 'yes'
+      'Would you like to see all the requirements before starting?',
+      'no', %w(yes no)
+    ) == 'yes'
       cli.puts ''
-      req_path = File.expand_path('utils/requirements.txt', MiGA.root_path)
+      req_path = File.join(MiGA.root_path, 'utils', 'requirements.txt')
       File.open(req_path, 'r') do |fh|
         fh.each_line { |ln| cli.puts ln }
       end
@@ -112,26 +120,28 @@ BANNER
     rc_path = File.expand_path('.miga_rc', ENV['HOME'])
     if File.exist? rc_path
       if cli.ask_user(
-            'I found a previous configuration. Do you want to continue?',
-            'yes', %w(yes no)) == 'no'
+        'I found a previous configuration. Do you want to continue?',
+        'yes', %w(yes no)
+      ) == 'no'
         cli.puts 'OK, see you soon!'
         exit(0)
       end
     end
     rc_fh = File.open(rc_path, 'w')
-    rc_fh.puts <<BASH
-#!/bin/bash
-# `miga init` made this on #{Time.now}
+    rc_fh.puts <<~BASH
+      #!/bin/bash
+      # `miga init` made this on #{Time.now}
 
-BASH
+    BASH
     rc_fh
   end
 
   def check_configuration_script(rc_fh)
     unless File.exist? cli[:config]
       cli[:config] = cli.ask_user(
-            'Is there a script I need to load at startup?',
-            cli[:config])
+        'Is there a script I need to load at startup?',
+        cli[:config]
+      )
     end
     if File.exist? cli[:config]
       cli[:config] = File.expand_path(cli[:config])
@@ -154,8 +164,10 @@ BASH
     File.open(req_path, 'r') do |fh|
       fh.each_line do |ln|
         next if $. < 3
+
         r = ln.chomp.split(/\t+/)
         next if r[0] =~ /\(opt\)$/ && !cli[:mytaxa]
+
         cli.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
         path = find_software(r[1])
         paths[r[1]] = File.expand_path(r[1], path).shellescape
@@ -200,18 +212,18 @@ BASH
   def check_additional_files(paths)
     if cli[:mytaxa]
       cli.puts 'Looking for MyTaxa databases:'
-      mt = File.dirname paths["MyTaxa"]
+      mt = File.dirname paths['MyTaxa']
       cli.print 'Looking for scores... '
       unless Dir.exist?(File.expand_path('db', mt))
-        cli.puts "no.\nExecute 'python2 #{mt}/utils/download_db.py'."
+        cli.puts "no\nExecute 'python2 #{mt}/utils/download_db.py'"
         exit(1)
       end
-      cli.puts 'yes.'
+      cli.puts 'yes'
       cli.print 'Looking for diamond db... '
       unless File.exist?(File.expand_path('AllGenomes.faa.dmnd', mt))
-        cli.puts "no.\nDownload " \
+        cli.puts "no\nDownload " \
           "'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" \
-          "AllGenomes.faa.dmnd' into #{mt}."
+          "AllGenomes.faa.dmnd' into #{mt}"
         exit(1)
       end
       cli.puts ''
@@ -223,7 +235,7 @@ BASH
     %w(ape cluster vegan).each do |pkg|
       cli.print "Testing #{pkg}... "
       if test_r_package(cli, paths, pkg)
-        cli.puts 'yes.'
+        cli.puts 'yes'
       else
         cli.puts 'no, installing'
         cli.print '' + install_r_package(cli, paths, pkg)
@@ -240,7 +252,7 @@ BASH
     %w(sqlite3 daemons json).each do |pkg|
       cli.print "Testing #{pkg}... "
       if test_ruby_gem(cli, paths, pkg)
-        cli.puts 'yes.'
+        cli.puts 'yes'
       else
         cli.puts 'no, installing'
         # This hackey mess is meant to ensure the test and installation are done
@@ -256,13 +268,13 @@ BASH
   end
 
   def close_rc_file(rc_fh)
-    rc_fh.puts <<FOOT
+    rc_fh.puts <<~FOOT
 
-MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
-MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
-MIGA_CONFIG_DATE='#{Time.now}'
+      MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
+      MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
+      MIGA_CONFIG_DATE='#{Time.now}'
 
-FOOT
+    FOOT
     rc_fh.close
   end
 end