miga-base 1.2.14.2 → 1.2.15.1

data/lib/miga/cli/action/doctor.rb

@@ -1,10 +1,15 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require 'miga/cli/action/doctor/base'
-
 class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
+  require 'miga/cli/action/doctor/base'
+  require 'miga/cli/action/doctor/databases'
+  require 'miga/cli/action/doctor/distances'
+  require 'miga/cli/action/doctor/operations'
   include MiGA::Cli::Action::Doctor::Base
+  include MiGA::Cli::Action::Doctor::Databases
+  include MiGA::Cli::Action::Doctor::Distances
+  include MiGA::Cli::Action::Doctor::Operations
 
   def parse_cli
     cli.defaults = { threads: 1 }
@@ -59,289 +64,4 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
       @@OPERATIONS
     end
   end
-
-  ##
-  # Perform status operation with MiGA::Cli +cli+
-  def check_status(cli)
-    cli.say 'Updating metadata status'
-    p = cli.load_project
-    n = p.dataset_names.size
-    (0 .. cli[:threads] - 1).map do |i|
-      Process.fork do
-        k = 0
-        cli.load_project.each_dataset do |d|
-          k += 1
-          cli.advance('Datasets:', k, n, false) if i == 0
-          d.recalculate_status if k % cli[:threads] == i
-        end
-      end
-    end
-    Process.waitall
-    cli.say
-  end
-
-  ##
-  # Perform databases operation with MiGA::Cli +cli+
-  def check_db(cli)
-    cli.say 'Checking integrity of databases'
-    p = cli.load_project
-    n = p.dataset_names.size
-    (0 .. cli[:threads] - 1).map do |i|
-      Process.fork do
-        k = 0
-        p.each_dataset do |d|
-          k += 1
-          cli.advance('Datasets:', k, n, false) if i == 0
-          next unless k % cli[:threads] == i
-          each_database_file(d) do |db_file, metric, result, _rank|
-            check_sqlite3_database(db_file, metric) do
-              cli.say(
-                "  > Removing malformed database from #{d.name}:#{result}   "
-              )
-              File.unlink(db_file)
-              r = d.result(result) or next
-              [r.path(:done), r.path].each do |f|
-                File.unlink(f) if File.exist?(f)
-              end
-            end
-          end
-        end
-      end
-    end
-    Process.waitall
-    cli.say
-  end
-
-  ##
-  # Perform bidirectional operation with MiGA::Cli +cli+
-  def check_bidir(cli)
-    cli.say 'Checking if reference distances are bidirectional'
-    project = cli.load_project
-    ref_ds = project.each_dataset.select(&:ref?)
-    ref_names = ref_ds.map(&:name)
-    n = ref_ds.size
-
-    # Read data first (threaded)
-    tmp = File.join(project.path, 'doctor-bidirectional.tmp')
-    FileUtils.mkdir_p(tmp)
-    MiGA::Parallel.process(cli[:threads]) do |thr|
-      file = File.join(tmp, "#{thr}.json")
-      fh = File.open(file, 'w')
-      [:aai, :ani].each do |metric|
-        fh.puts "# #{metric}"
-        ref_ds.each_with_index do |ds, idx|
-          if idx % cli[:threads] == thr
-            cli.advance('Reading:', idx + 1, n, false) if thr == 0
-            row = read_bidirectional(ds, metric)
-            fh.puts "#{ds.name} #{JSON.fast_generate(row)}" unless row.empty?
-          end
-        end
-      end
-      fh.puts '# end'
-      fh.flush # necessary for large threaded runs
-      fh.close
-      if thr == 0
-        cli.advance('Reading:', n, n, false)
-        cli.say
-      end
-    end
-
-    # Merge pieces per thread
-    dist = { aai: {}, ani: {} }
-    cli[:threads].times do |i|
-      cli.advance('Merging:', i + 1, cli[:threads], false)
-      file = File.join(tmp, "#{i}.json")
-      File.open(file, 'r') do |fh|
-        metric = nil
-        fh.each do |ln|
-          qry, row = ln.chomp.split(' ', 2)
-          if qry == '#'
-            metric = row.to_sym
-          else
-            raise "Unrecognized metric: #{metric}" unless dist[metric]
-            JSON.parse(row).each do |sbj, val|
-              dist[metric][qry] ||= {}
-              if dist[metric][sbj]&.include?(qry)
-                dist[metric][sbj].delete(qry) # Already bidirectional
-              else
-                dist[metric][qry][sbj] = val
-              end
-            end
-          end
-        end
-        raise "Incomplete thread dump: #{file}" unless metric == :end
-      end
-    end
-    cli.say
-    FileUtils.rm_rf(tmp)
-
-    # Write missing values (threaded)
-    MiGA::Parallel.distribute(ref_ds, cli[:threads]) do |ds, idx, thr|
-      cli.advance('Datasets:', idx + 1, n, false) if thr == 0
-      save_bidirectional(ds, dist)
-    end
-    cli.say
-  end
-
-  ##
-  # Perform distances operation with MiGA::Cli +cli+
-  def check_dist(cli)
-    p = cli.load_project
-    %i[ani aai].each do |dist|
-      res = p.result("#{dist}_distances")
-      next if res.nil?
-
-      cli.say "Checking #{dist} table for consistent datasets"
-      notok, fix = check_dist_eval(cli, p, res)
-      check_dist_fix(cli, p, fix)
-      check_dist_recompute(cli, res, notok)
-    end
-  end
-
-  ##
-  # Perform files operation with MiGA::Cli +cli+
-  def check_files(cli)
-    cli.say 'Looking for outdated files in results'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      d.each_result do |r_k, r|
-        ok = true
-        r.each_file do |_f_sym, _f_rel, f_abs|
-          unless File.exist? f_abs
-            ok = false
-            break
-          end
-        end
-        unless ok
-          cli.say "  > Registering again #{d.name}:#{r_k}   "
-          d.add_result(r_k, true, force: true)
-          sr = d.result(:stats) and sr.remove!
-        end
-      end
-    end
-    cli.say
-  end
-
-  ##
-  # Perform cds operation with MiGA::Cli +cli+
-  def check_cds(cli)
-    cli.say 'Looking for unzipped genes or proteins'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      res = d.result(:cds) or next
-      changed = false
-      %i[genes proteins gff3 gff2 tab].each do |f|
-        file = res.file_path(f) or next
-        if file !~ /\.gz/
-          cli.say "  > Gzipping #{d.name} #{f}   "
-          run_cmd(['gzip', '-9', file])
-          changed = true
-        end
-      end
-      if changed
-        d.add_result(:cds, true, force: true)
-        sr = d.result(:stats) and sr.remove!
-      end
-    end
-    cli.say
-  end
-
-  ##
-  # Perform essential-genes operation with MiGA::Cli +cli+
-  def check_ess(cli)
-    cli.say 'Looking for outdated essential genes'
-    cli.load_project.each_dataset do |d|
-      res = d.result(:essential_genes)
-      next if res.nil?
-
-      dir = res.file_path(:collection)
-      if dir.nil? || outdated_fastaai_ess(res)
-        cli.say "  > Removing #{d.name}:essential_genes"
-        res.remove!
-        d.result(:stats)&.remove!
-        next
-      end
-      next if Dir["#{dir}/*.faa"].empty?
-
-      cli.say "  > Fixing #{d.name}"
-      run_cmd <<~CMD
-        cd #{dir.shellescape} && tar -zcf proteins.tar.gz *.faa && rm *.faa
-      CMD
-    end
-  end
-
-  ##
-  # Check if the essential genes result +res+ has an outdated FastAAI index
-  def outdated_fastaai_ess(res)
-    idx1 = res.file_path(:fastaai_index)
-    idx2 = res.file_path(:fastaai_index_2)
-    idx2.nil? && !idx1.nil?
-  end
-
-  ##
-  # Perform mytaxa-scan operation with MiGA::Cli +cli+
-  def check_mts(cli)
-    cli.say 'Looking for unarchived MyTaxa Scan runs'
-    cli.load_project.each_dataset do |d|
-      res = d.result(:mytaxa_scan)
-      next if res.nil?
-
-      dir = res.file_path(:regions)
-      fix = false
-      unless dir.nil?
-        if Dir.exist? dir
-          run_cmd <<~CMD
-            cd #{dir.shellescape}/.. \
-                && tar -zcf '#{d.name}.reg.tar.gz' '#{d.name}.reg' \
-                && rm -r '#{d.name}.reg'
-          CMD
-        end
-        fix = true
-      end
-      %i[blast mytaxain wintax gene_ids region_ids].each do |ext|
-        file = res.file_path(ext)
-        unless file.nil?
-          FileUtils.rm(file) if File.exist? file
-          fix = true
-        end
-      end
-      if fix
-        cli.say "  > Fixing #{d.name}"
-        d.add_result(:mytaxa_scan, true, force: true)
-      end
-    end
-  end
-
-  ##
-  # Perform start operation with MiGA::Cli +cli+
-  def check_start(cli)
-    cli.say 'Looking for legacy .start files lingering'
-    cli.load_project.each_dataset do |d|
-      d.each_result do |r_k, r|
-        if File.exist? r.path(:start)
-          cli.say "  > Registering again #{d.name}:#{r_k}"
-          r.save
-        end
-      end
-    end
-  end
-
-  ##
-  # Perform taxonomy operation with MiGA::Cli +cli+
-  def check_tax(cli)
-    # cli.say 'o Checking for taxonomy/distances consistency'
-    # TODO: Find 95%ANI clusters with entries from different species
-    # TODO: Find different 95%ANI clusters with genomes from the same species
-    # TODO: Find AAI values too high or too low for each LCA rank
-  end
-
-  ##
-  # Run command +cmd+ with options +opts+
-  def run_cmd(cmd, opts = {})
-    opts = { return: :output, err2out: true, raise: false }.merge(opts)
-    cmdo = MiGA::MiGA.run_cmd(cmd, opts).chomp
-    warn(cmdo) unless cmdo.empty?
-  end
 end

data/lib/miga/cli/action/download/base.rb

@@ -7,6 +7,18 @@ end
 ##
 # Helper module including download functions for the *_get actions
 module MiGA::Cli::Action::Download::Base
+  def cli_base_flags(opt)
+    opt.on(
+      '--max INT', Integer,
+      'Maximum number of datasets to download (by default: unlimited)'
+    ) { |v| cli[:max_datasets] = v }
+    opt.on(
+      '-m', '--metadata STRING',
+      'Metadata as key-value pairs separated by = and delimited by comma',
+      'Values are saved as strings except for booleans (true / false) or nil'
+    ) { |v| cli[:metadata] = v }
+  end
+
   def cli_filters(opt)
     opt.on(
       '--exclude PATH',
@@ -17,6 +29,10 @@ module MiGA::Cli::Action::Download::Base
       '--ignore-until STRING',
       'Ignores all datasets until a name is found (useful for large reruns)'
     ) { |v| cli[:ignore_until] = v }
+    opt.on(
+      '--ignore-removed',
+      'Ignores entries removed from NCBI (by default fails on removed entries)'
+    ) { |v| cli[:ignore_removed] = v }
     cli.opt_flag(
       opt, 'get-metadata',
       'Only download and update metadata for existing datasets', :get_md
@@ -49,6 +65,40 @@ module MiGA::Cli::Action::Download::Base
     ) { |v| cli[:remote_list] = v }
   end
 
+  def generic_perform
+    p, ds = load_tasks
+    d, downloaded = download_entries(ds, p)
+
+    # Finalize
+    finalize_tasks(d, downloaded)
+    unlink_entries(p, p.dataset_names - d) if cli[:unlink]
+  end
+
+  def load_tasks
+    sanitize_cli
+    p = cli.load_project
+    ds = remote_list
+    ds = discard_excluded(ds)
+    ds = impose_limit(ds)
+    [p, ds]
+  end
+
+  def finalize_tasks(d, downloaded)
+    cli.say "Datasets listed: #{d.size}"
+    act = cli[:dry] ? 'to download' : 'downloaded'
+    cli.say "Datasets #{act}: #{downloaded}"
+    unless cli[:remote_list].nil?
+      File.open(cli[:remote_list], 'w') do |fh|
+        d.each { |i| fh.puts i }
+      end
+    end
+  end
+
+  def unlink_entries(p, unlink)
+    unlink.each { |i| p.unlink_dataset(i).remove! }
+    cli.say "Datasets unlinked: #{unlink.size}"
+  end
+
   def discard_excluded(ds)
     unless cli[:exclude].nil?
       cli.say "Discarding datasets in #{cli[:exclude]}"
@@ -84,7 +134,11 @@ module MiGA::Cli::Action::Download::Base
 
       downloaded += 1
       unless cli[:dry]
-        save_entry(name, body, p)
+        unless save_entry(name, body, p)
+          downloaded -= 1
+          d.pop
+          next
+        end
         p.save! if cli[:save_every] > 1 && (downloaded % cli[:save_every]).zero?
       end
     end
@@ -93,8 +147,11 @@ module MiGA::Cli::Action::Download::Base
     [d, downloaded]
   end
 
+  ##
+  # Saves the (generic remote) entry identified by +name+ with +body+ into the
+  # project +p+, and returns +true+ on success and +false+ otherwise
   def save_entry(name, body, p)
-    cli.say '  Locating remote dataset'
+    cli.say "  Locating remote dataset: #{name}"
     body[:md][:metadata_only] = true if cli[:only_md]
     rd = MiGA::RemoteDataset.new(body[:ids], body[:db], body[:universe])
     if cli[:get_md]
@@ -105,5 +162,10 @@ module MiGA::Cli::Action::Download::Base
       rd.save_to(p, name, !cli[:query], body[:md])
       cli.add_metadata(p.add_dataset(name))
     end
+    true
+  rescue MiGA::RemoteDataMissingError => e
+    raise(e) unless cli[:ignore_removed]
+    cli.say "    Removed dataset ignored: #{name}"
+    false
   end
 end

data/lib/miga/cli/action/gtdb_get.rb

@@ -18,15 +18,7 @@ class MiGA::Cli::Action::GtdbGet < MiGA::Cli::Action
         '-T', '--taxon STRING',
         '(Mandatory) Taxon name in GTDB format (e.g., g__Escherichia)'
       ) { |v| cli[:taxon] = v }
-      opt.on(
-        '--max INT', Integer,
-        'Maximum number of datasets to download (by default: unlimited)'
-      ) { |v| cli[:max_datasets] = v }
-      opt.on(
-        '-m', '--metadata STRING',
-        'Metadata as key-value pairs separated by = and delimited by comma',
-        'Values are saved as strings except for booleans (true / false) or nil'
-      ) { |v| cli[:metadata] = v }
+      cli_base_flags(opt)
       cli_task_flags(opt)
       cli_name_modifiers(opt)
       cli_filters(opt)
@@ -39,27 +31,6 @@ class MiGA::Cli::Action::GtdbGet < MiGA::Cli::Action
   end
 
   def perform
-    sanitize_cli
-    p = cli.load_project
-    ds = remote_list
-    ds = discard_excluded(ds)
-    ds = impose_limit(ds)
-    d, downloaded = download_entries(ds, p)
-
-    # Finalize
-    cli.say "Datasets listed: #{d.size}"
-    act = cli[:dry] ? 'to download' : 'downloaded'
-    cli.say "Datasets #{act}: #{downloaded}"
-    unless cli[:remote_list].nil?
-      File.open(cli[:remote_list], 'w') do |fh|
-        d.each { |i| fh.puts i }
-      end
-    end
-    return unless cli[:unlink]
-
-    unlink = p.dataset_names - d
-    unlink.each { |i| p.unlink_dataset(i).remove! }
-    cli.say "Datasets unlinked: #{unlink.size}"
+    generic_perform
   end
-
 end

data/lib/miga/cli/action/ncbi_get.rb

@@ -20,48 +20,23 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
         '-T', '--taxon STRING',
         '(Mandatory) Taxon name (e.g., a species binomial)'
       ) { |v| cli[:taxon] = v }
-      opt.on(
-        '--max INT', Integer,
-        'Maximum number of datasets to download (by default: unlimited)'
-      ) { |v| cli[:max_datasets] = v }
-      opt.on(
-        '-m', '--metadata STRING',
-        'Metadata as key-value pairs separated by = and delimited by comma',
-        'Values are saved as strings except for booleans (true / false) or nil'
-      ) { |v| cli[:metadata] = v }
+      cli_base_flags(opt)
       cli_task_flags(opt)
       cli_name_modifiers(opt)
       cli_filters(opt)
       cli_save_actions(opt)
       opt.on(
         '--api-key STRING',
+        '::HIDE::' # For backwards compatibility
+      ) { |v| ENV['NCBI_API_KEY'] = v }
+      opt.on(
+        '--ncbi-api-key STRING',
         'NCBI API key'
       ) { |v| ENV['NCBI_API_KEY'] = v }
     end
   end
 
   def perform
-    sanitize_cli
-    p = cli.load_project
-    ds = remote_list
-    ds = discard_excluded(ds)
-    ds = impose_limit(ds)
-    d, downloaded = download_entries(ds, p)
-
-    # Finalize
-    cli.say "Datasets listed: #{d.size}"
-    act = cli[:dry] ? 'to download' : 'downloaded'
-    cli.say "Datasets #{act}: #{downloaded}"
-    unless cli[:remote_list].nil?
-      File.open(cli[:remote_list], 'w') do |fh|
-        d.each { |i| fh.puts i }
-      end
-    end
-    return unless cli[:unlink]
-
-    unlink = p.dataset_names - d
-    unlink.each { |i| p.unlink_dataset(i).remove! }
-    cli.say "Datasets unlinked: #{unlink.size}"
+    generic_perform
   end
-
 end

data/lib/miga/cli/opt_helper.rb

@@ -43,7 +43,7 @@ module MiGA::Cli::OptHelper
       '-h', '--help',
       'Display this screen'
     ) do
-      puts opt
+      puts opt.to_s.gsub(/^.*\s+::HIDE::\s*$/, '')
       exit
     end
     opt.separator ''

data/lib/miga/common/errors.rb

@@ -9,4 +9,14 @@ module MiGA
   # An error with a system call
   class SystemCallError < Error
   end
+
+  ##
+  # An error with remote data
+  class RemoteDataError < Error
+  end
+
+  ##
+  # An error caused by missing remote data
+  class RemoteDataMissingError < RemoteDataError
+  end
 end

data/lib/miga/dataset/base.rb

@@ -7,18 +7,47 @@ class MiGA::Dataset < MiGA::MiGA
 
   # Class-level
   class << self
+    ##
+    # Directories containing the results from dataset-specific tasks
     def RESULT_DIRS
       @@RESULT_DIRS
     end
 
+    ##
+    # Supported dataset types
     def KNOWN_TYPES
       @@KNOWN_TYPES
     end
 
+    ##
+    # Returns an Array of tasks (Symbols) to be executed before project-wide
+    # tasks
     def PREPROCESSING_TASKS
       @@PREPROCESSING_TASKS
     end
 
+    ##
+    # Tasks to be excluded from query datasets
+    def EXCLUDE_NOREF_TASKS
+      @@EXCLUDE_NOREF_TASKS
+    end
+
+    ##
+    # Tasks to be executed only in datasets that are single-organism. These
+    # tasks are ignored for multi-organism datasets or for unknown types
+    def ONLY_NONMULTI_TASKS
+      @@ONLY_NONMULTI_TASKS
+    end
+
+    ##
+    # Tasks to be executed only in datasets that are multi-organism. These
+    # tasks are ignored for single-organism datasets or for unknwon types
+    def ONLY_MULTI_TASKS
+      @@ONLY_MULTI_TASKS
+    end
+
+    ##
+    # Options supported by datasets
     def OPTIONS
       @@OPTIONS
     end
@@ -69,7 +98,7 @@ module MiGA::Dataset::Base
   }
 
   ##
-  # Returns an Array of tasks to be executed before project-wide tasks
+  # Returns an Array of tasks (Symbols) to be executed before project-wide tasks
   @@PREPROCESSING_TASKS = [
     :raw_reads, :trimmed_reads, :read_quality, :trimmed_fasta,
     :assembly, :cds, :essential_genes, :mytaxa, :mytaxa_scan,
@@ -77,19 +106,19 @@ module MiGA::Dataset::Base
   ]
 
   ##
-  # Tasks to be excluded from query datasets.
+  # Tasks to be excluded from query datasets
   @@EXCLUDE_NOREF_TASKS = [:mytaxa_scan, :taxonomy]
   @@_EXCLUDE_NOREF_TASKS_H = Hash[@@EXCLUDE_NOREF_TASKS.map { |i| [i, true] }]
 
   ##
-  # Tasks to be executed only in datasets that are not multi-organism. These
-  # tasks are ignored for multi-organism datasets or for unknown types.
+  # Tasks to be executed only in datasets that are single-organism. These
+  # tasks are ignored for multi-organism datasets or for unknown types
   @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :taxonomy, :distances]
   @@_ONLY_NONMULTI_TASKS_H = Hash[@@ONLY_NONMULTI_TASKS.map { |i| [i, true] }]
 
   ##
   # Tasks to be executed only in datasets that are multi-organism. These
-  # tasks are ignored for single-organism datasets or for unknwon types.
+  # tasks are ignored for single-organism datasets or for unknwon types
   @@ONLY_MULTI_TASKS = [:mytaxa]
   @@_ONLY_MULTI_TASKS_H = Hash[@@ONLY_MULTI_TASKS.map { |i| [i, true] }]