miga-base 0.7.24.0 → 0.7.26.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82ca84b468df712ab38eba74071c26e8dcb1d87335764dde2ba86075abc8c8d5
-  data.tar.gz: 36583e298fc020faa5cc16af2f6a3fcc88c30619a4f529fd6f03772f6c437bb9
+  metadata.gz: dda80476da72166bd5eb6bc22a5ae7afc94285a58afe312633a06115e7336d80
+  data.tar.gz: 176c5a3c167232bb221b778b2bb3762c6795893171740dbff83b899d68034cd9
 SHA512:
-  metadata.gz: d78a3709fecba4594d8dc9a2351f217b3605022237e424eb1a194ccf62ddf2059eb1df274f68a15cda74936dfece3f2123485483773172cc62cfba9ff454c5d2
-  data.tar.gz: fbf11b04c062701b204f4ba93425dea6f8ff1492fe48f7182472e65b16fccbde98e86d396a79cf51c6e410a246c3b9e9509043eaf952b320191ec086e2efa573
+  metadata.gz: 90b69b23c515e7f8ce2508b5c719856112e2b874876c81bc18767fa5b384d3d6b2e1b819b313f7afbc6d9260ddbaf9b143bd3503c641eed29f1738197837c13b
+  data.tar.gz: f25b7224d1b37c386e0d4776a40be0a05496bfa9fd9715c59722794e298405280b3043ac26f7cbd5d033090673776b93a7744610aa24fd0bf7331329f0561e64

data/lib/miga/cli/action/add.rb

@@ -6,10 +6,7 @@ require 'miga/cli/action'
 class MiGA::Cli::Action::Add < MiGA::Cli::Action
   def parse_cli
     cli.expect_files = true
-    cli.defaults = {
-      ref: true, ignore_dups: false,
-      regexp: MiGA::Cli.FILE_REGEXP
-    }
+    cli.defaults = { ref: true, ignore_dups: false }
     cli.parse do |opt|
       opt.separator 'You can create multiple datasets with a single command; ' \
         'simply pass all the files at the end: {FILES...}'
@@ -37,7 +34,10 @@ class MiGA::Cli::Action::Add < MiGA::Cli::Action
       opt.on(
         '-R', '--name-regexp REGEXP', Regexp,
         'Regular expression indicating how to extract the name from the path',
-        "By default: '#{cli[:regexp]}'"
+        'By default for paired files:',
+        "'#{MiGA::Cli.FILE_REGEXP(true)}'",
+        'By default for other files:',
+        "'#{MiGA::Cli.FILE_REGEXP}'"
       ) { |v| cli[:regexp] = v }
       opt.on(
         '--prefix STRING',
@@ -59,6 +59,9 @@ class MiGA::Cli::Action::Add < MiGA::Cli::Action
     p = cli.load_project
     files, file_type = get_files_and_type
 
+    paired = cli[:input_type].to_s.include?('_paired')
+    cli[:regexp] ||= MiGA::Cli.FILE_REGEXP(paired)
+
     cli.say 'Creating datasets:'
     files.each do |file|
       d = create_dataset(file, p)
@@ -166,7 +169,7 @@ class MiGA::Cli::Action::Add < MiGA::Cli::Action
     file_type[2].each_with_index do |ext, i|
       gz = file[i] =~ /\.gz/ ? '.gz' : ''
       FileUtils.cp(file[i], "#{r_path}#{ext}#{gz}")
-      cli.say "  file: #{file[i]}"
+      cli.say "  file: #{File.basename(file[i])}"
     end
     File.open("#{r_path}.done", 'w') { |f| f.print Time.now.to_s }
   end

data/lib/miga/cli/action/derep_wf.rb

@@ -55,7 +55,7 @@ class MiGA::Cli::Action::DerepWf < MiGA::Cli::Action
       { run_project_stats: false, run_clades: false },
       { run_mytaxa_scan: false, run_ssu: false }
     )
-    p.set_option(:gsp_metric, cli[:metric])
+    p.set_option(:gsp_metric, cli[:metric].to_s)
     p.set_option(:"gsp_#{cli[:metric]}", cli[:threshold])
 
     # Run

data/lib/miga/cli/action/index_wf.rb

@@ -16,8 +16,10 @@ class MiGA::Cli::Action::IndexWf < MiGA::Cli::Action
         'Perform MyTaxa scan analysis'
       ) { |v| cli[:mytaxa] = v }
       opts_for_wf_distances(opt)
-      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)',
-                  cleanup: false, project_type: true)
+      opts_for_wf(
+        opt, 'Input genome assemblies (nucleotides, FastA)',
+        cleanup: false, project_type: true
+      )
     end
   end
 

data/lib/miga/cli/action/init.rb

@@ -14,6 +14,8 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     cli.defaults = {
       mytaxa: nil,
       rdp: nil,
+      reads: nil,
+      optional: nil,
       config: File.join(ENV['MIGA_HOME'], '.miga_modules'),
       ask: false,
       auto: false,
@@ -35,6 +37,16 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
         'Should I try setting up the RDP classifier?',
         'By default: interactive (true if --auto)'
       ) { |v| cli[:rdp] = v }
+      opt.on(
+        '--[no-]read-processing',
+        'Should I try setting up read processing software?',
+        'By default: interactive (true if --auto)'
+      ) { |v| cli[:reads] = v }
+      opt.on(
+        '--[no-]optional',
+        'Should I try setting up the optional software?',
+        'Automatically sets answers for mytaxa, rdp, and reads'
+      ) { |v| cli[:optional] = v }
       opt.on(
         '--daemon-type STRING',
         'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
@@ -61,8 +73,7 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     check_configuration_script(rc_fh)
     paths = check_software_requirements(rc_fh)
     check_additional_files(paths)
-    check_r_packages(paths)
-    check_ruby_gems(paths)
+    check_libraries(paths)
     configure_daemon
     close_rc_file(rc_fh)
     cli.puts 'Configuration complete. MiGA is ready to work!'
@@ -83,34 +94,6 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     )
   end
 
-  def test_r_package(cli, paths, pkg)
-    run_r_cmd(cli, paths, "library('#{pkg}')")
-    $?.success?
-  end
-
-  def install_r_package(cli, paths, pkg)
-    r_cmd = "install.packages('#{pkg}', repos='http://cran.rstudio.com/')"
-    run_r_cmd(cli, paths, r_cmd)
-  end
-
-  def test_ruby_gem(cli, paths, pkg)
-    run_cmd(
-      cli,
-      "#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null"
-    )
-    $?.success?
-  end
-
-  def install_ruby_gem(cli, paths, pkg)
-    gem_cmd = "Gem::GemRunner.new.run %w(install --user #{pkg})"
-    run_cmd(
-      cli,
-      "#{paths['ruby'].shellescape} \
-        -r rubygems -r rubygems/gem_runner \
-        -e #{gem_cmd.shellescape} 2>&1"
-    )
-  end
-
   def list_requirements
     if cli.ask_user(
       'Would you like to see all the requirements before starting?',
@@ -133,6 +116,8 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     rc_fh.puts "export MIGA_MYTAXA='#{cli[:mytaxa] ? 'yes' : 'no'}'"
     ask_for_optional(:rdp, 'RDP classifier')
     rc_fh.puts "export MIGA_RDP='#{cli[:rdp] ? 'yes' : 'no'}'"
+    ask_for_optional(:reads, 'read processing')
+    rc_fh.puts "export MIGA_READS='#{cli[:reads] ? 'yes' : 'no'}'"
     paths = {}
     rc_fh.puts 'MIGA_PATH=""'
     req_path = File.expand_path('utils/requirements.txt', MiGA.root_path)
@@ -152,20 +137,20 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
   def define_software(ln)
     r = ln.chomp.split(/\t+/)
     return if %w[Software --------].include?(r[0])
-    return if r[0] =~ /\(mytaxa\)$/ && !cli[:mytaxa]
-    return if r[0] =~ /\(rdp\)$/ && !cli[:rdp]
+    %i[mytaxa rdp reads].each { |i| return if r[0] =~ /\(#{i}\)$/ && !cli[i] }
 
     r
   end
 
   def ask_for_optional(symbol, name)
-    if cli[symbol].nil?
-      cli[symbol] =
-        cli.ask_user(
-          "Should I include #{name} modules?",
-          'yes', %w(yes no)
-        ) == 'yes'
-    end
+    cli[symbol] = cli[:optional] if !cli[:optional].nil? && cli[symbol].nil?
+    return cli[symbol] unless cli[symbol].nil?
+
+    cli[symbol] =
+      cli.ask_user(
+        "Should I include #{name} modules?",
+        'yes', %w(yes no)
+      ) == 'yes'
   end
 
   def find_software(exec)
@@ -189,40 +174,70 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     path
   end
 
-  def check_r_packages(paths)
-    cli.puts 'Looking for R packages:'
-    %w(ape cluster vegan).each do |pkg|
-      cli.print "Testing #{pkg}... "
-      if test_r_package(cli, paths, pkg)
-        cli.puts 'yes'
-      else
-        cli.puts 'no, installing'
-        cli.print '' + install_r_package(cli, paths, pkg)
-        unless test_r_package(cli, paths, pkg)
-          raise "Unable to auto-install R package: #{pkg}"
-        end
+  def check_libraries(paths)
+    req_libraries = {
+      r: %w[ape cluster vegan],
+      ruby: %w[sqlite3 daemons json],
+      python: %w[numpy]
+    }
+
+    req_libraries.each do |language, libraries|
+      cli.puts "Looking for #{language.to_s.capitalize} libraries:"
+      libraries.each do |lib|
+        check_and_install_library(paths, language, lib)
       end
+      cli.puts ''
     end
-    cli.puts ''
   end
 
-  def check_ruby_gems(paths)
-    cli.puts 'Looking for Ruby gems:'
-    %w(sqlite3 daemons json).each do |pkg|
-      cli.print "Testing #{pkg}... "
-      if test_ruby_gem(cli, paths, pkg)
-        cli.puts 'yes'
-      else
-        cli.puts 'no, installing'
-        # This hackey mess is meant to ensure the test and installation are done
-        # on the configuration Ruby, not on the Ruby currently executing the
-        # init action
-        cli.print install_ruby_gem(cli, paths, pkg)
-        unless test_ruby_gem(cli, paths, pkg)
-          raise "Unable to auto-install Ruby gem: #{pkg}"
-        end
+  def check_and_install_library(paths, language, library)
+    cli.print "Testing #{library}... "
+    if test_library(cli, paths, language, library)
+      cli.puts 'yes'
+    else
+      cli.puts 'no, installing'
+      cli.print '' + install_library(cli, paths, language, library)
+      unless test_library(cli, paths, language, library)
+        raise "Cannot install #{language.to_s.capitalize} library: #{library}"
       end
     end
-    cli.puts ''
+  end
+
+  def test_library(cli, paths, language, pkg)
+    case language
+    when :r
+      run_r_cmd(cli, paths, "library('#{pkg}')")
+    when :ruby
+      x = "#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null"
+      run_cmd(cli, x)
+    when :python
+      x = "#{paths['python3'].shellescape} -c 'import #{pkg}' 2>/dev/null"
+      run_cmd(cli, x)
+    else
+      raise "Unrecognized language: #{language}"
+    end
+    $?.success?
+  end
+
+  def install_library(cli, paths, language, pkg)
+    case language
+    when :r
+      r_cmd = "install.packages('#{pkg}', repos='http://cran.rstudio.com/')"
+      run_r_cmd(cli, paths, r_cmd)
+    when :ruby
+      # This hackey mess is meant to ensure the test and installation are done
+      # on the configuration Ruby, not on the Ruby currently executing the
+      # init action
+      gem_cmd = "Gem::GemRunner.new.run %w(install --user #{pkg})"
+      x = "#{paths['ruby'].shellescape} -r rubygems -r rubygems/gem_runner \
+            -e #{gem_cmd.shellescape} 2>&1"
+      run_cmd(cli, x)
+    when :python
+      x = "#{paths['python3'].shellescape} \
+            -m pip install --user #{pkg.shellescape} 2>&1"
+      run_cmd(cli, x)
+    else
+      raise "Unrecognized language: #{language}"
+    end
   end
 end

data/lib/miga/cli/action/init/files_helper.rb

@@ -26,7 +26,7 @@ module MiGA::Cli::Action::Init::FilesHelper
   def close_rc_file(rc_fh)
     rc_fh.puts <<~FOOT
 
-      MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
+      MIGA_CONFIG_VERSION='#{MiGA::MiGA.FULL_VERSION}'
       MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
       MIGA_CONFIG_DATE='#{Time.now}'
 
@@ -59,6 +59,7 @@ module MiGA::Cli::Action::Init::FilesHelper
     end
     check_rdp_classifier if cli[:rdp]
     check_phyla_lite
+    cli.puts ''
   end
 
   def check_mytaxa_scores(paths)

data/lib/miga/cli/action/option.rb

@@ -17,6 +17,14 @@ class MiGA::Cli::Action::Option < MiGA::Cli::Action
         'Value of the option to set (by default, option value is not changed)',
         'Recognized tokens: nil, true, false'
       ) { |v| cli[:value] = v }
+      opt.on(
+        '--about',
+        'Print additional information about the values supported by this option'
+      ) { |v| cli[:about] = v }
+      opt.on(
+        '--tab',
+        'Return a tab-delimited table'
+      ) { |v| cli[:tabular] = v }
       opt.on(
         '-o', '--output PATH',
         'Create output file instead of returning to STDOUT'
@@ -28,13 +36,24 @@ class MiGA::Cli::Action::Option < MiGA::Cli::Action
     unless cli[:value].nil?
       cli.ensure_par(
         { key: '-k' },
-        '%<name>s is mandatory when --value is set: please provide %<flag>s'
+        '%<name>s is mandatory when --value is set: provide %<flag>s'
       )
     end
     obj = cli.load_project_or_dataset
     io = cli[:output].nil? ? $stdout : File.open(cli[:output], 'w')
     if cli[:key].nil?
-      cli.table(%w[Key Value], obj.all_options.to_a, io)
+      opts = obj.all_options
+                .map { |k, v| [k, v, obj.assert_has_option(k)[:desc]] }
+      cli.table(%w[Key Value Definition], opts, io)
+    elsif cli[:about]
+      opt = obj.assert_has_option(cli[:key])
+      title = "#{cli[:key]}: #{opt[:desc]}"
+      io.puts title
+      io.puts '-' * title.length
+      opt.each do |k, v|
+        v = v[obj] if v.is_a? Proc
+        io.puts "#{k.to_s.capitalize}: #{v}" unless k == :desc
+      end
     else
       obj.set_option(cli[:key], cli[:value], true) unless cli[:value].nil?
       io.puts obj.option(cli[:key])

data/lib/miga/cli/action/preproc_wf.rb

@@ -21,18 +21,20 @@ class MiGA::Cli::Action::PreprocWf < MiGA::Cli::Action
         '-m', '--mytaxa_scan',
         'Perform MyTaxa scan analysis'
       ) { |v| cli[:mytaxa] = v }
-      opts_for_wf(opt, 'Input files as defined by --input-type',
-                  multi: true, cleanup: false, ncbi: false)
+      opts_for_wf(
+        opt, 'Input files as defined by --input-type',
+        multi: true, cleanup: false, ncbi: false, project_type: true
+      )
     end
   end
 
   def perform
     # Input data
     cli.ensure_par(input_type: '-i')
-    p_metadata = Hash[
-      %w[project_stats haai_distances aai_distances ani_distances clade_finding]
-                 .map { |i| ["run_#{i}", false] }
+    norun = %w[
+      project_stats haai_distances aai_distances ani_distances clade_finding
     ]
+    p_metadata = Hash[norun.map { |i| ["run_#{i}", false] }]
     d_metadata = { run_distances: false }
     unless cli[:mytaxa]
       d_metadata[:run_mytaxa_scan] = false

data/lib/miga/cli/action/wf.rb

@@ -7,8 +7,7 @@ module MiGA::Cli::Action::Wf
   def default_opts_for_wf
     cli.expect_files = true
     cli.defaults = {
-      clean: false, regexp: MiGA::Cli.FILE_REGEXP,
-      project_type: :genomes, dataset_type: :popgenome,
+      clean: false, project_type: :genomes, dataset_type: :popgenome,
       ncbi_draft: true, min_qual: MiGA::Project.OPTIONS[:min_qual][:default]
     }
   end
@@ -56,28 +55,10 @@ module MiGA::Cli::Action::Wf
     opt.on(
       '-R', '--name-regexp REGEXP', Regexp,
       'Regular expression indicating how to extract the name from the path',
-      "By default: '#{cli[:regexp]}'"
+      "By default: '#{MiGA::Cli.FILE_REGEXP}'"
     ) { |v| cli[:regexp] = v }
-    opt.on(
-      '--type STRING',
-      "Type of datasets. By default: #{cli[:dataset_type]}",
-      'Recognized types:',
-      *MiGA::Dataset.KNOWN_TYPES
-        .map do |k, v|
-          "~ #{k}: #{v[:description]}" unless !params[:multi] && v[:multi]
-        end.compact
-    ) { |v| cli[:dataset_type] = v.downcase.to_sym }
-    if params[:project_type]
-      opt.on(
-        '--project-type STRING',
-        "Type of project. By default: #{cli[:project_type]}",
-        'Recognized types:',
-        *MiGA::Project.KNOWN_TYPES
-          .map do |k, v|
-            "~ #{k}: #{v[:description]}" unless !params[:multi] && v[:multi]
-          end.compact
-      ) { |v| cli[:project_type] = v.downcase.to_sym }
-    end
+    opt_object_type(opt, :dataset, params[:multi])
+    opt_object_type(opt, :project, params[:multi]) if params[:project_type]
     opt.on(
       '--daemon PATH',
       'Use custom daemon configuration in JSON format',
@@ -106,7 +87,7 @@ module MiGA::Cli::Action::Wf
     end
     opt.on(
       '--haai-p STRING',
-      'hAAI search engine. One of: blast+ (default), blat, diamond, no'
+      'hAAI search engine. One of: blast+ (default), fastaai, blat, diamond, no'
     ) { |v| cli[:haai_p] = v }
     opt.on(
       '--aai-p STRING',
@@ -124,10 +105,14 @@ module MiGA::Cli::Action::Wf
       project_type: '--project-type',
       dataset_type: '--dataset-type'
     )
+    paired = cli[:input_type].to_s.include?('_paired')
+    cli[:regexp] ||= MiGA::Cli.FILE_REGEXP(paired)
+
     # Create empty project
     call_cli(
       ['new', '-P', cli[:outdir], '-t', cli[:project_type]]
     ) unless MiGA::Project.exist? cli[:outdir]
+
     # Define project metadata
     p = cli.load_project(:outdir, '-o')
     p_metadata[:type] = cli[:project_type]
@@ -135,6 +120,7 @@ module MiGA::Cli::Action::Wf
     %i[haai_p aai_p ani_p ess_coll min_qual].each do |i|
       p.set_option(i, cli[i])
     end
+
     # Download datasets
     unless cli[:ncbi_taxon].nil?
       what = cli[:ncbi_draft] ? '--all' : '--complete'
@@ -142,6 +128,7 @@ module MiGA::Cli::Action::Wf
         ['ncbi_get', '-P', cli[:outdir], '-T', cli[:ncbi_taxon], what]
       )
     end
+
     # Add datasets
     call_cli(
       [
@@ -153,6 +140,7 @@ module MiGA::Cli::Action::Wf
         '-R', cli[:regexp]
       ] + cli.files
     ) unless cli.files.empty?
+
     # Define datasets metadata
     p.load
     d_metadata[:type] = cli[:dataset_type]
@@ -208,4 +196,32 @@ module MiGA::Cli::Action::Wf
     md.each { |k, v| obj.metadata[k] = v }
     obj.save
   end
+
+  private
+
+  ##
+  # Add option --type or --project-type to +opt+
+  def opt_object_type(opt, obj, multi)
+    conf =
+      case obj
+      when :dataset
+        ['type', 'datasets', :dataset_type, MiGA::Dataset]
+      when :project
+        ['project-type', 'project', :project_type, MiGA::Project]
+      else
+        raise "Unrecognized object type: #{obj}"
+      end
+
+    options =
+      conf[3].KNOWN_TYPES.map do |k, v|
+        "~ #{k}: #{v[:description]}" unless !multi && v[:multi]
+      end.compact
+
+    opt.on(
+      "--#{conf[0]} STRING",
+      "Type of #{conf[1]}. By default: #{cli[conf[2]]}",
+      'Recognized types:',
+      *options
+    ) { |v| cli[conf[2]] = v.downcase.to_sym }
+  end
 end