miga-base 1.2.14.2 → 1.2.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8a1b62d1125f54550d8cf801c46924abb79085005b8fa4abe732e9347b1fa82
-  data.tar.gz: baa4a96d604fd649147f47aafeaea038310b8ec588edb8055f7d511842ba3f6b
+  metadata.gz: 0ad52d75f0a01d681043530335e20e89ecfb46822a8166dd7c0993b9822882f9
+  data.tar.gz: 28bf47b3e257718cb3071ebf48d2f0fd63a8797116f8108faa789643940aac3f
 SHA512:
-  metadata.gz: 1ab3bed5edbd0e9ef513c731c9d28999bdb483547376446746157b58bddc92a9ddde7e0cdfd7766351f11e69f1e11df8e2ccf94815ab95108842caa43f6fdd29
-  data.tar.gz: b3401886ca9139d3ed3a9c19188a1b171254c0d0c1c21d63beffb71b497713dac59ba5fb0176bf2340bc48595640dc6dadfba7881aa7c993cd82894b0679c8a6
+  metadata.gz: b498f1617e597feb77f6adc655f7fdd0ee941b670867f5157f12f5af49a462c712a72ea60a7b6ef28ad702a4afdcca0a33bac00216d306ef4bff2ddcee4f844b
+  data.tar.gz: 41755ac99346b7dbbdf1a971158f98c99ca970d4134b912b17989d4776ae528902fed6df68d7f477fecfcc02657fae0350f311e3195c2211bc4e0945744366c8

data/lib/miga/cli/action/doctor/base.rb

@@ -1,47 +1,7 @@
 require 'miga/cli/action'
 require 'miga/sqlite'
 
-class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
-end
-
 module MiGA::Cli::Action::Doctor::Base
-  ##
-  # Check the database in +db_file+ maintains integrity for the
-  # tables saving +metric+ (:ani or :aai) and call +blk+ if the
-  # file is corrupt or doesn't contain the expected structure
-  def check_sqlite3_database(db_file, metric, &blk)
-    MiGA::SQLite.new(db_file).run("select count(*) from #{metric}")
-  rescue SQLite3::SQLException, SQLite3::CorruptException
-    blk.call
-  end
-
-  def each_database_file(dataset, &blk)
-    ref_db = {
-      haai: ['01.haai', :aai], aai: ['02.aai', :aai], ani: ['03.ani', :ani]
-    }
-    qry_db = {
-      haai: ['.haai.db', :aai], aai: ['.aai.db', :aai], ani: ['.ani.db', :ani]
-    }
-    base = File.join(dataset.project.path, 'data', '09.distances')
-    result = :distances
-    if dataset.ref?
-      file_db = "#{dataset.name}.db"
-      ref_db.each do |rank, v|
-        dir, metric = *v
-        file = File.join(base, dir, file_db)
-        blk[file, metric, result, rank] if File.exist? file
-      end
-      # Query databases for reference databases refer to taxonomy runs
-      base = File.join(base, '05.taxonomy')
-      result = :taxonomy
-    end
-    qry_db.each do |rank, v|
-      ext, metric = *v
-      file = File.join(base, "#{dataset.name}#{ext}")
-      blk[file, metric, result, rank] if File.exist? file
-    end
-  end
-
   ##
   # Scans the all-vs-all matrix registered in +res+ (MiGA::Result) in search of
   # pairs where one or both datasets are missing or inactive in the project +p+
@@ -157,4 +117,20 @@ module MiGA::Cli::Action::Doctor::Base
       end
     end
   end
+
+  ##
+  # Run command +cmd+ with options +opts+
+  def run_cmd(cmd, opts = {})
+    opts = { return: :output, err2out: true, raise: false }.merge(opts)
+    cmdo = MiGA::MiGA.run_cmd(cmd, opts).chomp
+    warn(cmdo) unless cmdo.empty?
+  end
+
+  ##
+  # Check if the essential genes result +res+ has an outdated FastAAI index
+  def outdated_fastaai_ess(res)
+    idx1 = res.file_path(:fastaai_index)
+    idx2 = res.file_path(:fastaai_index_2)
+    idx2.nil? && !idx1.nil?
+  end
 end

data/lib/miga/cli/action/doctor/databases.rb

@@ -0,0 +1,39 @@
+
+module MiGA::Cli::Action::Doctor::Databases
+  ##
+  # Check the database in +db_file+ maintains integrity for the
+  # tables saving +metric+ (:ani or :aai) and call +blk+ if the
+  # file is corrupt or doesn't contain the expected structure
+  def check_sqlite3_database(db_file, metric, &blk)
+    MiGA::SQLite.new(db_file).run("select count(*) from #{metric}")
+  rescue SQLite3::SQLException, SQLite3::CorruptException
+    blk.call
+  end
+
+  def each_database_file(dataset, &blk)
+    ref_db = {
+      haai: ['01.haai', :aai], aai: ['02.aai', :aai], ani: ['03.ani', :ani]
+    }
+    qry_db = {
+      haai: ['.haai.db', :aai], aai: ['.aai.db', :aai], ani: ['.ani.db', :ani]
+    }
+    base = File.join(dataset.project.path, 'data', '09.distances')
+    result = :distances
+    if dataset.ref?
+      file_db = "#{dataset.name}.db"
+      ref_db.each do |rank, v|
+        dir, metric = *v
+        file = File.join(base, dir, file_db)
+        blk[file, metric, result, rank] if File.exist? file
+      end
+      # Query databases for reference databases refer to taxonomy runs
+      base = File.join(base, '05.taxonomy')
+      result = :taxonomy
+    end
+    qry_db.each do |rank, v|
+      ext, metric = *v
+      file = File.join(base, "#{dataset.name}#{ext}")
+      blk[file, metric, result, rank] if File.exist? file
+    end
+  end
+end

data/lib/miga/cli/action/doctor/distances.rb

@@ -0,0 +1,144 @@
+
+module MiGA::Cli::Action::Doctor::Distances
+  ##
+  # Perform databases operation with MiGA::Cli +cli+
+  def check_db(cli)
+    cli.say 'Checking integrity of databases'
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        p.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
+          each_database_file(d) do |db_file, metric, result, _rank|
+            check_sqlite3_database(db_file, metric) do
+              cli.say(
+                "  > Removing malformed database from #{d.name}:#{result}   "
+              )
+              File.unlink(db_file)
+              r = d.result(result) or next
+              [r.path(:done), r.path].each do |f|
+                File.unlink(f) if File.exist?(f)
+              end
+            end
+          end
+        end
+      end
+    end
+    Process.waitall
+    cli.say
+  end
+
+  ##
+  # Perform bidirectional operation with MiGA::Cli +cli+
+  def check_bidir(cli)
+    cli.say 'Checking if reference distances are bidirectional'
+    project = cli.load_project
+    ref_ds = project.each_dataset.select(&:ref?)
+
+    # Read and merge data
+    tmp = partial_bidir_tmp(project, ref_ds)
+    dist = merge_bidir_tmp(tmp)
+    FileUtils.rm_rf(tmp)
+
+    # Write missing values (threaded)
+    MiGA::Parallel.distribute(ref_ds, cli[:threads]) do |ds, idx, thr|
+      cli.advance('Datasets:', idx + 1, ref_ds.size, false) if thr == 0
+      save_bidirectional(ds, dist)
+    end
+    cli.say
+  end
+
+  ##
+  # Perform distances operation with MiGA::Cli +cli+
+  def check_dist(cli)
+    p = cli.load_project
+    %i[ani aai].each do |dist|
+      res = p.result("#{dist}_distances")
+      next if res.nil?
+
+      cli.say "Checking #{dist} table for consistent datasets"
+      notok, fix = check_dist_eval(cli, p, res)
+      check_dist_fix(cli, p, fix)
+      check_dist_recompute(cli, res, notok)
+    end
+  end
+
+  #---- Auxuliary functions -----
+
+  ##
+  # Make a temporal directory holding partial bidirectionality reports (one per thread)
+  # in a custom multi-JSON format. Requires a MiGA::Project +project+ and the iterator of
+  # the reference datasets +ref_ds+. Returns the path to the temporal directory created.
+  # Used by +check_bidir+
+  def partial_bidir_tmp(project, ref_ds)
+    n = ref_ds.size
+
+    # Read data first (threaded)
+    tmp = File.join(project.path, 'doctor-bidirectional.tmp')
+    FileUtils.mkdir_p(tmp)
+    MiGA::Parallel.process(cli[:threads]) do |thr|
+      file = File.join(tmp, "#{thr}.json")
+      fh = File.open(file, 'w')
+      [:aai, :ani].each do |metric|
+        fh.puts "# #{metric}"
+        ref_ds.each_with_index do |ds, idx|
+          if idx % cli[:threads] == thr
+            cli.advance('Reading:', idx + 1, n, false) if thr == 0
+            row = read_bidirectional(ds, metric)
+            fh.puts "#{ds.name} #{JSON.fast_generate(row)}" unless row.empty?
+          end
+        end
+      end
+      fh.puts '# end'
+      fh.flush # necessary for large threaded runs
+      fh.close
+      if thr == 0
+        cli.advance('Reading:', n, n, false)
+        cli.say
+      end
+    end
+
+    return tmp
+  end
+
+  ##
+  # Read partial temporal reports of bidirectionality (located in +tmp+), and return
+  # a two-deep hash with the final missingness report by metric (first key) and
+  # dataset name (second key). Used by +check_bidir+
+  def merge_bidir_tmp(tmp)
+    dist = { aai: {}, ani: {} }
+    cli[:threads].times do |i|
+      cli.advance('Merging:', i + 1, cli[:threads], false)
+      file = File.join(tmp, "#{i}.json")
+      File.open(file, 'r') do |fh|
+        metric = nil
+        fh.each do |ln|
+          qry, row = ln.chomp.split(' ', 2)
+          row or raise "Unexpected format in #{file}:#{$.}"
+          if qry == '#'
+            metric = row.to_sym
+          else
+            raise "Unrecognized metric: #{metric}" unless dist[metric]
+            JSON.parse(row).each do |sbj, val|
+              dist[metric][qry] ||= {}
+              if dist[metric][sbj]&.include?(qry)
+                dist[metric][sbj].delete(qry) # Already bidirectional
+              else
+                dist[metric][qry][sbj] = val
+              end
+            end
+          end
+        end
+        raise "Incomplete thread dump: #{file}" unless metric == :end
+      end
+    end
+    cli.say
+
+    return dist
+  end
+end
+

data/lib/miga/cli/action/doctor/operations.rb

@@ -0,0 +1,159 @@
+
+module MiGA::Cli::Action::Doctor::Operations
+  ##
+  # Perform status operation with MiGA::Cli +cli+
+  def check_status(cli)
+    cli.say 'Updating metadata status'
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        cli.load_project.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          d.recalculate_status if k % cli[:threads] == i
+        end
+      end
+    end
+    Process.waitall
+    cli.say
+  end
+
+  # check_db in Distances
+
+  # check_bidir in Distances
+
+  # check_dist in Distances
+
+  ##
+  # Perform files operation with MiGA::Cli +cli+
+  def check_files(cli)
+    cli.say 'Looking for outdated files in results'
+    n, k = cli.load_project.dataset_names.size, 0
+    cli.load_project.each_dataset do |d|
+      cli.advance('Datasets:', k += 1, n, false)
+      d.each_result do |r_k, r|
+        ok = true
+        r.each_file do |_f_sym, _f_rel, f_abs|
+          unless File.exist? f_abs
+            ok = false
+            break
+          end
+        end
+        unless ok
+          cli.say "  > Registering again #{d.name}:#{r_k}   "
+          d.add_result(r_k, true, force: true)
+          sr = d.result(:stats) and sr.remove!
+        end
+      end
+    end
+    cli.say
+  end
+
+  ##
+  # Perform cds operation with MiGA::Cli +cli+
+  def check_cds(cli)
+    cli.say 'Looking for unzipped genes or proteins'
+    n, k = cli.load_project.dataset_names.size, 0
+    cli.load_project.each_dataset do |d|
+      cli.advance('Datasets:', k += 1, n, false)
+      res = d.result(:cds) or next
+      changed = false
+      %i[genes proteins gff3 gff2 tab].each do |f|
+        file = res.file_path(f) or next
+        if file !~ /\.gz/
+          cli.say "  > Gzipping #{d.name} #{f}   "
+          run_cmd(['gzip', '-9', file])
+          changed = true
+        end
+      end
+      if changed
+        d.add_result(:cds, true, force: true)
+        sr = d.result(:stats) and sr.remove!
+      end
+    end
+    cli.say
+  end
+
+  ##
+  # Perform essential-genes operation with MiGA::Cli +cli+
+  def check_ess(cli)
+    cli.say 'Looking for outdated essential genes'
+    cli.load_project.each_dataset do |d|
+      res = d.result(:essential_genes)
+      next if res.nil?
+
+      dir = res.file_path(:collection)
+      if dir.nil? || outdated_fastaai_ess(res)
+        cli.say "  > Removing #{d.name}:essential_genes"
+        res.remove!
+        d.result(:stats)&.remove!
+        next
+      end
+      next if Dir["#{dir}/*.faa"].empty?
+
+      cli.say "  > Fixing #{d.name}"
+      run_cmd <<~CMD
+        cd #{dir.shellescape} && tar -zcf proteins.tar.gz *.faa && rm *.faa
+      CMD
+    end
+  end
+
+  ##
+  # Perform mytaxa-scan operation with MiGA::Cli +cli+
+  def check_mts(cli)
+    cli.say 'Looking for unarchived MyTaxa Scan runs'
+    cli.load_project.each_dataset do |d|
+      res = d.result(:mytaxa_scan)
+      next if res.nil?
+
+      dir = res.file_path(:regions)
+      fix = false
+      unless dir.nil?
+        if Dir.exist? dir
+          run_cmd <<~CMD
+            cd #{dir.shellescape}/.. \
+                && tar -zcf '#{d.name}.reg.tar.gz' '#{d.name}.reg' \
+                && rm -r '#{d.name}.reg'
+          CMD
+        end
+        fix = true
+      end
+      %i[blast mytaxain wintax gene_ids region_ids].each do |ext|
+        file = res.file_path(ext)
+        unless file.nil?
+          FileUtils.rm(file) if File.exist? file
+          fix = true
+        end
+      end
+      if fix
+        cli.say "  > Fixing #{d.name}"
+        d.add_result(:mytaxa_scan, true, force: true)
+      end
+    end
+  end
+
+  ##
+  # Perform start operation with MiGA::Cli +cli+
+  def check_start(cli)
+    cli.say 'Looking for legacy .start files lingering'
+    cli.load_project.each_dataset do |d|
+      d.each_result do |r_k, r|
+        if File.exist? r.path(:start)
+          cli.say "  > Registering again #{d.name}:#{r_k}"
+          r.save
+        end
+      end
+    end
+  end
+
+  ##
+  # Perform taxonomy operation with MiGA::Cli +cli+
+  def check_tax(cli)
+    # cli.say 'o Checking for taxonomy/distances consistency'
+    # TODO: Find 95%ANI clusters with entries from different species
+    # TODO: Find different 95%ANI clusters with genomes from the same species
+    # TODO: Find AAI values too high or too low for each LCA rank
+  end
+end