miga-base 0.7.26.0 → 1.0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dda80476da72166bd5eb6bc22a5ae7afc94285a58afe312633a06115e7336d80
-  data.tar.gz: 176c5a3c167232bb221b778b2bb3762c6795893171740dbff83b899d68034cd9
+  metadata.gz: b4d9aad1ec5368e6cce82c4385a1d40839e948115e7749709a51e9c517d762f0
+  data.tar.gz: dc7b6500cbd85a1c50ef92fd41c0436dbbc16273ad245f730f8c2beb684db587
 SHA512:
-  metadata.gz: 90b69b23c515e7f8ce2508b5c719856112e2b874876c81bc18767fa5b384d3d6b2e1b819b313f7afbc6d9260ddbaf9b143bd3503c641eed29f1738197837c13b
-  data.tar.gz: f25b7224d1b37c386e0d4776a40be0a05496bfa9fd9715c59722794e298405280b3043ac26f7cbd5d033090673776b93a7744610aa24fd0bf7331329f0561e64
+  metadata.gz: 5973a0b393ff61b1065b74ae36f580be0e80467c8c5b281e82541539b9a97d37b553d82d4d284ec041be72937f7de1274cfa94f2f1c8593a3dcb4c97d3820eff
+  data.tar.gz: 60388895b6d864154ff94cd70ae7959f2ac80688a6f46c1d092fc8b08c7597ef3814bdf947bcc05a9832a9a54136c61a4a1970c4aefa3b05d57ad9c6bb670c1d

data/lib/miga/cli/action/classify_wf.rb

@@ -50,7 +50,7 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
     # Input data
     ref_db = reference_db
     norun = %w[
-      project_stats haai_distances aai_distances ani_distances clade_finding
+      haai_distances aai_distances ani_distances clade_finding
     ]
     p_metadata = Hash[norun.map { |i| ["run_#{i}", false] }]
     p = create_project(
@@ -59,7 +59,7 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
       run_ssu: false, run_mytaxa_scan: false, run_distances: false
     )
     p.set_option(:ref_project, ref_db.path)
-    p.set_option(:tax_pvalue, cli[:pvalue], true)
+    p.set_option(:tax_pvalue, cli[:pvalue])
     # Run
     run_daemon
     summarize(%w[cds assembly essential_genes]) if cli[:summaries]

data/lib/miga/cli/action/derep_wf.rb

@@ -52,7 +52,7 @@ class MiGA::Cli::Action::DerepWf < MiGA::Cli::Action
     # Input data
     p = create_project(
       :assembly,
-      { run_project_stats: false, run_clades: false },
+      { run_clades: false },
       { run_mytaxa_scan: false, run_ssu: false }
     )
     p.set_option(:gsp_metric, cli[:metric].to_s)

data/lib/miga/cli/action/doctor.rb

@@ -93,7 +93,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
           k += 1
           cli.advance('Datasets:', k, n, false) if i == 0
           next unless k % cli[:threads] == i
-          each_database_file(d) do |db_file, metric, result|
+          each_database_file(d) do |db_file, metric, result, _rank|
             check_sqlite3_database(db_file, metric) do
               cli.say(
                 "  > Removing malformed database from #{d.name}:#{result}   "
@@ -116,27 +116,70 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform bidirectional operation with MiGA::Cli +cli+
   def check_bidir(cli)
     cli.say 'Checking if reference distances are bidirectional'
-    ref_ds = cli.load_project.each_dataset.select(&:ref?)
+    project = cli.load_project
+    ref_ds = project.each_dataset.select(&:ref?)
     ref_names = ref_ds.map(&:name)
     n = ref_ds.size
-    (0 .. cli[:threads] - 1).map do |i|
-      Process.fork do
-        k = 0
-        ref_ds.each do |d|
-          k += 1
-          cli.advance('Datasets:', k, n, false) if i == 0
-          next unless k % cli[:threads] == i
 
-          saved = saved_targets(d)
-          next if saved.nil?
+    # Read data first (threaded)
+    tmp = File.join(project.path, 'doctor-bidirectional.tmp')
+    FileUtils.mkdir_p(tmp)
+    MiGA::Parallel.process(cli[:threads]) do |thr|
+      file = File.join(tmp, "#{thr}.json")
+      fh = File.open(file, 'w')
+      [:aai, :ani].each do |metric|
+        fh.puts "# #{metric}"
+        ref_ds.each_with_index do |ds, idx|
+          if idx % cli[:threads] == thr
+            cli.advance('Reading:', idx + 1, n, false) if thr == 0
+            row = read_bidirectional(ds, metric)
+            fh.puts "#{ds.name} #{JSON.fast_generate(row)}" unless row.empty?
+          end
+        end
+      end
+      fh.puts '# end'
+      fh.flush # necessary for large threaded runs
+      fh.close
+      if thr == 0
+        cli.advance('Reading:', n, n, false)
+        cli.say
+      end
+    end
 
-          (ref_names - saved).each do |k|
-            save_bidirectional(cli.load_project.dataset(k), d)
+    # Merge pieces per thread
+    dist = { aai: {}, ani: {} }
+    cli[:threads].times do |i|
+      cli.advance('Merging:', i + 1, cli[:threads], false)
+      file = File.join(tmp, "#{i}.json")
+      File.open(file, 'r') do |fh|
+        metric = nil
+        fh.each do |ln|
+          qry, row = ln.chomp.split(' ', 2)
+          if qry == '#'
+            metric = row.to_sym
+          else
+            raise "Unrecognized metric: #{metric}" unless dist[metric]
+            JSON.parse(row).each do |sbj, val|
+              dist[metric][qry] ||= {}
+              if dist[metric][sbj]&.include?(qry)
+                dist[metric][sbj].delete(qry) # Already bidirectional
+              else
+                dist[metric][qry][sbj] = val
+              end
+            end
           end
         end
+        raise "Incomplete thread dump: #{file}" unless metric == :end
       end
     end
-    Process.waitall
+    cli.say
+    FileUtils.rm_rf(tmp)
+
+    # Write missing values (threaded)
+    MiGA::Parallel.distribute(ref_ds, cli[:threads]) do |ds, idx, thr|
+      cli.advance('Datasets:', idx + 1, n, false) if thr == 0
+      save_bidirectional(ds, dist)
+    end
     cli.say
   end
 

data/lib/miga/cli/action/doctor/base.rb

@@ -16,22 +16,28 @@ module MiGA::Cli::Action::Doctor::Base
   end
 
   def each_database_file(dataset, &blk)
-    ref_db = { '01.haai' => :aai, '02.aai' => :aai, '03.ani' => :ani }
-    qry_db = { '.haai.db' => :aai, '.aai.db' => :aai, '.ani.db' => :ani }
+    ref_db = {
+      haai: ['01.haai', :aai], aai: ['02.aai', :aai], ani: ['03.ani', :ani]
+    }
+    qry_db = {
+      haai: ['.haai.db', :aai], aai: ['.aai.db', :aai], ani: ['.ani.db', :ani]
+    }
     base = File.join(dataset.project.path, 'data', '09.distances')
     result = :distances
     if dataset.ref?
       file_db = "#{dataset.name}.db"
-      ref_db.each do |dir, metric|
+      ref_db.each do |rank, v|
+        dir, metric = *v
         file = File.join(base, dir, file_db)
-        blk[file, metric, result] if File.exist? file
+        blk[file, metric, result, rank] if File.exist? file
       end
       base = File.join(base, '05.taxonomy')
       result = :taxonomy
     end
-    qry_db.each do |ext, metric|
+    qry_db.each do |rank, v|
+      ext, metric = *v
       file = File.join(base, "#{dataset.name}#{ext}")
-      blk[file, metric, result] if File.exist? file
+      blk[file, metric, result, rank] if File.exist? file
     end
   end
 
@@ -109,25 +115,43 @@ module MiGA::Cli::Action::Doctor::Base
   end
 
   ##
-  # Saves all the distance estimates in +a+ -> +b+ into the +b+ databases
-  # (as +b+ -> +a+), where both +a+ and +b+ are MiGA::Dataset objects
-  def save_bidirectional(a, b)
-    each_database_file(a) do |db_file, metric, result|
-      data = nil
-      data = MiGA::SQLite.new(db_file).run(
-        "select seq1, seq2, #{metric}, sd, n, omega " +
-        "from #{metric} where seq2 = ? limit 1", b.name
-      ).first
-      next if data.nil? || data.empty?
+  # Reads all the distance estimates in +a+ -> * for +metric+ and
+  # returns them as a hash +{"b_name" => [val, sd, ...], ...}+
+  def read_bidirectional(a, metric)
+    db_file = a.result(:distances)&.file_path("#{metric}_db") or return {}
+    sql = "select seq2, #{metric}, sd, n, omega from #{metric}"
+    data = MiGA::SQLite.new(db_file).run(sql) || []
+    Hash[
+      data.map do |row|
+        k, v = row.shift(2)
+        [k, row.all?(&:zero?) ? v : [v] + row]
+      end
+    ]
+  end
 
-      db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
-      next unless File.exist?(db_file_b)
+  ##
+  # Saves all the distance estimates in * -> +a+ into the +a+ databases
+  # (as +a+ -> *), where +a+ is a MiGA::Dataset object, with currently
+  # saved values read from the hash +dist+
+  def save_bidirectional(a, dist)
+    each_database_file(a) do |db_file, metric, result, rank|
+      next if rank == :haai # No need for hAAI to be bidirectional
 
-      data[0], data[1] = data[1], data[0]
-      MiGA::SQLite.new(db_file_b).run(
-        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-        "values(?, ?, ?, ?, ?, ?)", data
-      )
+      b2a = dist[rank].map { |b_name, v| b_name if v[a.name] }.compact
+      a2b = dist[rank][a.name]&.keys || []
+      SQLite3::Database.new(db_file) do |db|
+        sql = <<~SQL
+          insert into #{metric}(seq1, seq2, #{metric}, sd, n, omega) \
+          values(?, ?, ?, ?, ?, ?);
+        SQL
+        db.execute('BEGIN TRANSACTION;')
+        (b2a - a2b).each do |b_name|
+          val = dist[rank][b_name][a.name]
+          val = [val, 0, 0, 0] unless val.is_a?(Array)
+          db.execute(sql, [a.name, b_name] + val)
+        end
+        db.execute('COMMIT;')
+      end
     end
   end
 end

data/lib/miga/cli/action/init.rb

@@ -112,12 +112,15 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
 
   def check_software_requirements(rc_fh)
     cli.puts 'Looking for requirements:'
-    ask_for_optional(:mytaxa, 'MyTaxa')
-    rc_fh.puts "export MIGA_MYTAXA='#{cli[:mytaxa] ? 'yes' : 'no'}'"
-    ask_for_optional(:rdp, 'RDP classifier')
-    rc_fh.puts "export MIGA_RDP='#{cli[:rdp] ? 'yes' : 'no'}'"
-    ask_for_optional(:reads, 'read processing')
-    rc_fh.puts "export MIGA_READS='#{cli[:reads] ? 'yes' : 'no'}'"
+    opt_groups = {
+      mytaxa: 'MyTaxa',
+      rdp: 'RDP classifier',
+      reads: 'read processing'
+    }
+    opt_groups.each do |k, v|
+      ask_for_optional(k, v)
+      rc_fh.puts "export MIGA_#{k.to_s.upcase}='#{cli[k] ? 'yes' : 'no'}'"
+    end
     paths = {}
     rc_fh.puts 'MIGA_PATH=""'
     req_path = File.expand_path('utils/requirements.txt', MiGA.root_path)
@@ -196,8 +199,9 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
       cli.puts 'yes'
     else
       cli.puts 'no, installing'
-      cli.print '' + install_library(cli, paths, language, library)
+      out = install_library(cli, paths, language, library)
       unless test_library(cli, paths, language, library)
+        cli.puts out
         raise "Cannot install #{language.to_s.capitalize} library: #{library}"
       end
     end

data/lib/miga/cli/action/init/files_helper.rb

@@ -83,6 +83,7 @@ module MiGA::Cli::Action::Init::FilesHelper
       cli.puts 'yes'
     elsif File.exist?(mt_db)
       cli.puts 'yes, sym-linking'
+      FileUtils.mkdir_p(miga_db)
       File.symlink(mt_db, home_db)
     else
       cli.puts 'no, downloading'

data/lib/miga/cli/action/ncbi_get.rb

@@ -18,7 +18,7 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
       cli.opt_object(opt, [:project])
       opt.on(
         '-T', '--taxon STRING',
-        '(Mandatory unless --reference) Taxon name (e.g., a species binomial)'
+        '(Mandatory) Taxon name (e.g., a species binomial)'
       ) { |v| cli[:taxon] = v }
       opt.on(
         '-m', '--metadata STRING',
@@ -137,7 +137,7 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
   end
 
   def sanitize_cli
-    cli.ensure_par(taxon: '-T') unless cli[:reference]
+    cli.ensure_par(taxon: '-T')
     tasks = %w[reference complete chromosome scaffold contig]
     unless tasks.any? { |i| cli[i.to_sym] }
       raise 'No action requested: pick at least one type of genome'
@@ -204,7 +204,7 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
         'from(GenomeAssemblies).' \
         'usingschema(/schema/GenomeAssemblies).' \
         'matching(tab==["Prokaryotes"] and q=="' \
-          "#{cli[:taxon].tr('"', "'")}\"",
+          "#{cli[:taxon]&.tr('"', "'")}\"",
       fields: 'organism|organism,assembly|assembly,replicons|replicons,' \
         'level|level,ftp_path_genbank|ftp_path_genbank,' \
         'release_date|release_date,strain|strain',

data/lib/miga/cli/action/tax_dist.rb

@@ -99,13 +99,13 @@ class MiGA::Cli::Action::TaxDist < MiGA::Cli::Action
       ds_name = []
       File.open(tab, 'r') do |fh|
         fh.each_line do |ln|
-          if ln =~ /^ {#{(rank_i - 1) * 2}}\S+:\S+:/
+          if ln =~ /^ {0,#{(rank_i - 1) * 2}}\S+:\S+:/
             in_rank = nil
             ds_name = []
           elsif ln =~ /^ {#{rank_i * 2}}(#{rank}:(\S+)):/
             in_rank = $2 == '?' ? nil : $1
             ds_name = []
-          elsif ln =~ /^ *# (\S+)/ and not in_rank.nil?
+          elsif ln =~ /^ *# (\S+)/ && !in_rank.nil?
             ds_i = $1
             ds_name << ds_i
             ds_name.each do |ds_j|

data/lib/miga/cli/action/wf.rb

@@ -81,21 +81,22 @@ module MiGA::Cli::Action::Wf
       cli[:aai_p] = 'blast+'
       cli[:ani_p] = 'blast+'
     end
-    opt.on('--fast', 'Alias to: --aai-p diamond --ani-p fastani') do
+    opt.on('--fast', 'Alias to: --aai-p diamond --ani-p fastani (default)') do
       cli[:aai_p] = 'diamond'
       cli[:ani_p] = 'fastani'
     end
     opt.on(
       '--haai-p STRING',
-      'hAAI search engine. One of: blast+ (default), fastaai, blat, diamond, no'
+      'hAAI search engine. One of: blast+, fastaai, blat, diamond, fastaai, no',
+      'The default is "no" for clade projects and "fastaai" otherwise'
     ) { |v| cli[:haai_p] = v }
     opt.on(
       '--aai-p STRING',
-      'AAI search engine. One of: blast+ (default), blat, diamond'
+      'AAI search engine. One of: blast+, blat, diamond (default)'
     ) { |v| cli[:aai_p] = v }
     opt.on(
       '--ani-p STRING',
-      'ANI search engine. One of: blast+ (default), blat, fastani'
+      'ANI search engine. One of: blast+, blat, fastani (default)'
     ) { |v| cli[:ani_p] = v }
   end
 

data/lib/miga/common.rb

@@ -3,6 +3,7 @@
 
 require 'miga/version'
 require 'miga/json'
+require 'miga/parallel'
 require 'miga/common/base'
 require 'miga/common/path'
 require 'miga/common/format'

data/lib/miga/daemon.rb

@@ -73,10 +73,10 @@ class MiGA::Daemon < MiGA::MiGA
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
     miga_say "Saving log to: #{output_file}" unless show_log?
-    queue_maintenance
-    load_status
     say 'Configuration options:'
     say @runopts.to_s
+    load_status
+    queue_maintenance(true)
   end
 
   ##
@@ -87,6 +87,7 @@ class MiGA::Daemon < MiGA::MiGA
     check_datasets or check_project
     if shutdown_when_done? && (jobs_running.size + jobs_to_run.size).zero?
       say 'Nothing else to do, shutting down'
+      exit_cleanup
       return false
     end
     flush!
@@ -102,13 +103,19 @@ class MiGA::Daemon < MiGA::MiGA
 
   ##
   # Queue maintenance tasks as an analysis job
-  def queue_maintenance
-    return if bypass_maintenance? || shutdown_when_done?
+  def queue_maintenance(force = false)
+    return if bypass_maintenance? || (!force && shutdown_when_done?)
 
     say 'Queueing maintenance tasks'
     queue_job(:maintenance)
   end
 
+  ##
+  # Remove temporary files on completion
+  def exit_cleanup
+    FileUtils.rm_f(File.join(daemon_home, 'status.json'))
+  end
+
   ##
   # Send +msg+ to +say+ as long as +level+ is at most +verbosity+
   def l_say(level, *msg)

data/lib/miga/dataset/result.rb

@@ -181,26 +181,30 @@ module MiGA::Dataset::Result
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       if result_files_exist?(base, '.2.clipped.fastq')
-        {
-          pair1: '.1.clipped.fastq',
-          pair2: '.2.clipped.fastq',
-          single: '.1.clipped.single.fastq'
-        }
+        { pair1: '.1.clipped.fastq', pair2: '.2.clipped.fastq' }
       else
         { single: '.1.clipped.fastq' }
       end
     ).tap do |r|
+      # Legacy files
       r.add_file(:trimming_sumary, "#{name}.1.fastq.trimmed.summary.txt")
+      r.add_file(:single, "#{name}.1.clipped.single.fastq")
     end
   end
 
   ##
   # Add result type +:read_quality+ at +base+ (no +_opts+ supported)
   def add_result_read_quality(base, _opts)
-    return nil unless result_files_exist?(base, %w[.solexaqa .fastqc])
+    return nil unless
+      result_files_exist?(base, %w[.post.1.html]) ||
+      result_files_exist?(base, %w[.solexaqa .fastqc])
 
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
+      pre_qc_1: '.pre.1.html', pre_qc_2: '.pre.2.html',
+      post_qc_1: '.post.1.html', post_qc_2: '.post.2.html',
+      adapter_detection: '.adapters.txt',
+      # Legacy files
       solexaqa: '.solexaqa', fastqc: '.fastqc'
     )
   end

data/lib/miga/json.rb

@@ -1,5 +1,4 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
 
 require 'json'
 
@@ -45,8 +44,10 @@ class MiGA::Json < MiGA::MiGA
       raise "Empty descriptor: #{opts[:contents] ? "''" : path}" if cont.empty?
 
       # Parse JSON
-      params = { symbolize_names: opts[:symbolize],
-                 create_additions: opts[:additions] }
+      params = {
+        symbolize_names: opts[:symbolize],
+        create_additions: opts[:additions]
+      }
       y = JSON.parse(cont, params)
 
       # Add defaults