miga-base 0.7.26.3 → 1.0.0.sr1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f2c9f2013b77f3a3b1dff0fbb73e74dad0b836d17d4a2129110a6095ffc7d8a
-  data.tar.gz: 0d7a1c879970dc9ffae5bae8cbbb51f6ef7ef13e41373fcc7cd5cc93db841104
+  metadata.gz: a599c1e0c51a62f7303cbd1bbf9f8568649dbbeae768b518ad67250b1c3217a4
+  data.tar.gz: f8300cb5c44209d8a3639338319b50cdbce01cf14362006a9f61147854625bd9
 SHA512:
-  metadata.gz: 4da1c02b538370a585efc788a71cbb9fafb5b8383c2882f0ab01d1bdec22a9dac2878bb1fa3a028f78ff19d22ecf70e50c67e20e71dc8a0308437ca19a6368db
-  data.tar.gz: 4eb38dafd4eb0dbaf69eed6fa734fec5c0d104d97d1f035cafdcd77365f93ecc723ac01eda643986f0676c4b4156a2504f92c1d98a02d0bd3955306e1a4924f5
+  metadata.gz: 4bf676ee04f650e8f2388b0f0e732e9da941ca13ceb6b02d986800d353adf2de261ab61a56dde496210f2965255ca0ff9df2cbf5927f8643ddd706736511ef07
+  data.tar.gz: b5cffa3afdb384db2a7fd2d86d11f062491d3df403569cae8bb2209df9119012e539179453efc492227835553bade1ff03805d8b26be4b5988890a7bc9684475

data/lib/miga/cli/action/doctor.rb

@@ -116,38 +116,69 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform bidirectional operation with MiGA::Cli +cli+
   def check_bidir(cli)
     cli.say 'Checking if reference distances are bidirectional'
-    ref_ds = cli.load_project.each_dataset.select(&:ref?)
+    project = cli.load_project
+    ref_ds = project.each_dataset.select(&:ref?)
     ref_names = ref_ds.map(&:name)
     n = ref_ds.size
 
     # Read data first (threaded)
-    @distances = { aai: {}, ani: {} }
-    Dir.mktmpdir do |tmp|
-      MiGA::Parallel.process(cli[:threads]) do |thr|
-        idx = 0
-        ref_ds.each do |ds|
-          cli.advance('Reading:', idx + 1, n, false) if thr == 0
-          read_bidirectional(ds) if idx % cli[:threads] == thr
-          idx += 1
-        end
-        File.open("#{tmp}/#{thr}.json", 'w') do |fh|
-          fh.print JSON.fast_generate(@distances)
+    tmp = File.join(project.path, 'doctor-bidirectional.tmp')
+    FileUtils.mkdir_p(tmp)
+    MiGA::Parallel.process(cli[:threads]) do |thr|
+      file = File.join(tmp, "#{thr}.json")
+      fh = File.open(file, 'w')
+      [:aai, :ani].each do |metric|
+        fh.puts "# #{metric}"
+        ref_ds.each_with_index do |ds, idx|
+          if idx % cli[:threads] == thr
+            cli.advance('Reading:', idx + 1, n, false) if thr == 0
+            row = read_bidirectional(ds, metric)
+            fh.puts "#{ds.name} #{JSON.fast_generate(row)}" unless row.empty?
+          end
         end
       end
-      cli.say
+      fh.puts '# end'
+      fh.flush # necessary for large threaded runs
+      fh.close
+      if thr == 0
+        cli.advance('Reading:', n, n, false)
+        cli.say
+      end
+    end
 
-      cli[:threads].times do |i|
-        cli.advance('Merging:', i + 1, cli[:threads], false)
-        o = MiGA::Json.parse("#{tmp}/#{i}.json", symbolize: false)
-        o.each { |k, v| @distances[k.to_sym].merge!(v) }
+    # Merge pieces per thread
+    dist = { aai: {}, ani: {} }
+    cli[:threads].times do |i|
+      cli.advance('Merging:', i + 1, cli[:threads], false)
+      file = File.join(tmp, "#{i}.json")
+      File.open(file, 'r') do |fh|
+        metric = nil
+        fh.each do |ln|
+          qry, row = ln.chomp.split(' ', 2)
+          if qry == '#'
+            metric = row.to_sym
+          else
+            raise "Unrecognized metric: #{metric}" unless dist[metric]
+            JSON.parse(row).each do |sbj, val|
+              dist[metric][qry] ||= {}
+              if dist[metric][sbj]&.include?(qry)
+                dist[metric][sbj].delete(qry) # Already bidirectional
+              else
+                dist[metric][qry][sbj] = val
+              end
+            end
+          end
+        end
+        raise "Incomplete thread dump: #{file}" unless metric == :end
       end
-      cli.say
     end
+    cli.say
+    FileUtils.rm_rf(tmp)
 
     # Write missing values (threaded)
     MiGA::Parallel.distribute(ref_ds, cli[:threads]) do |ds, idx, thr|
       cli.advance('Datasets:', idx + 1, n, false) if thr == 0
-      save_bidirectional(ds)
+      save_bidirectional(ds, dist)
     end
     cli.say
   end

data/lib/miga/cli/action/doctor/base.rb

@@ -115,30 +115,30 @@ module MiGA::Cli::Action::Doctor::Base
   end
 
   ##
-  # Reads all the distance estimates in +a+ -> *, and saves them in memory
-  # in the +@distances+ variable.
-  def read_bidirectional(a)
-    each_database_file(a) do |db_file, metric, result, rank|
-      next if rank == :haai # No need for hAAI to be bidirectional
-
-      sql = "select seq2, #{metric}, sd, n, omega from #{metric}"
-      data = MiGA::SQLite.new(db_file).run(sql)
-      next if data.nil? || data.empty?
-
-      @distances[rank][a.name] ||= {}
-      data.each { |row| @distances[rank][a.name][row.shift] = row }
-    end
+  # Reads all the distance estimates in +a+ -> * for +metric+ and
+  # returns them as a hash +{"b_name" => [val, sd, ...], ...}+
+  def read_bidirectional(a, metric)
+    db_file = a.result(:distances)&.file_path("#{metric}_db") or return {}
+    sql = "select seq2, #{metric}, sd, n, omega from #{metric}"
+    data = MiGA::SQLite.new(db_file).run(sql) || []
+    Hash[
+      data.map do |row|
+        k, v = row.shift(2)
+        [k, row.all?(&:zero?) ? v : [v] + row]
+      end
+    ]
   end
 
   ##
   # Saves all the distance estimates in * -> +a+ into the +a+ databases
-  # (as +a+ -> *), where +a+ is a MiGA::Dataset object
-  def save_bidirectional(a)
+  # (as +a+ -> *), where +a+ is a MiGA::Dataset object, with currently
+  # saved values read from the hash +dist+
+  def save_bidirectional(a, dist)
     each_database_file(a) do |db_file, metric, result, rank|
       next if rank == :haai # No need for hAAI to be bidirectional
 
-      b2a = @distances[rank].map { |b_name, v| b_name if v[a.name] }.compact
-      a2b = @distances[rank][a.name].keys
+      b2a = dist[rank].map { |b_name, v| b_name if v[a.name] }.compact
+      a2b = dist[rank][a.name]&.keys || []
       SQLite3::Database.new(db_file) do |db|
         sql = <<~SQL
           insert into #{metric}(seq1, seq2, #{metric}, sd, n, omega) \
@@ -146,7 +146,9 @@ module MiGA::Cli::Action::Doctor::Base
         SQL
         db.execute('BEGIN TRANSACTION;')
         (b2a - a2b).each do |b_name|
-          db.execute(sql, [a.name, b_name] + @distances[rank][b_name][a.name])
+          val = dist[rank][b_name][a.name]
+          val = [val, 0, 0, 0] unless val.is_a?(Array)
+          db.execute(sql, [a.name, b_name] + val)
         end
         db.execute('COMMIT;')
       end

data/lib/miga/cli/action/init.rb

@@ -112,12 +112,15 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
 
   def check_software_requirements(rc_fh)
     cli.puts 'Looking for requirements:'
-    ask_for_optional(:mytaxa, 'MyTaxa')
-    rc_fh.puts "export MIGA_MYTAXA='#{cli[:mytaxa] ? 'yes' : 'no'}'"
-    ask_for_optional(:rdp, 'RDP classifier')
-    rc_fh.puts "export MIGA_RDP='#{cli[:rdp] ? 'yes' : 'no'}'"
-    ask_for_optional(:reads, 'read processing')
-    rc_fh.puts "export MIGA_READS='#{cli[:reads] ? 'yes' : 'no'}'"
+    opt_groups = {
+      mytaxa: 'MyTaxa',
+      rdp: 'RDP classifier',
+      reads: 'read processing'
+    }
+    opt_groups.each do |k, v|
+      ask_for_optional(k, v)
+      rc_fh.puts "export MIGA_#{k.to_s.upcase}='#{cli[k] ? 'yes' : 'no'}'"
+    end
     paths = {}
     rc_fh.puts 'MIGA_PATH=""'
     req_path = File.expand_path('utils/requirements.txt', MiGA.root_path)
@@ -196,8 +199,9 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
       cli.puts 'yes'
     else
       cli.puts 'no, installing'
-      cli.print '' + install_library(cli, paths, language, library)
+      out = install_library(cli, paths, language, library)
       unless test_library(cli, paths, language, library)
+        cli.puts out
         raise "Cannot install #{language.to_s.capitalize} library: #{library}"
       end
     end

data/lib/miga/cli/action/init/files_helper.rb

@@ -83,6 +83,7 @@ module MiGA::Cli::Action::Init::FilesHelper
       cli.puts 'yes'
     elsif File.exist?(mt_db)
       cli.puts 'yes, sym-linking'
+      FileUtils.mkdir_p(miga_db)
       File.symlink(mt_db, home_db)
     else
       cli.puts 'no, downloading'

data/lib/miga/cli/action/ncbi_get.rb

@@ -18,7 +18,7 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
       cli.opt_object(opt, [:project])
       opt.on(
         '-T', '--taxon STRING',
-        '(Mandatory unless --reference) Taxon name (e.g., a species binomial)'
+        '(Mandatory) Taxon name (e.g., a species binomial)'
       ) { |v| cli[:taxon] = v }
       opt.on(
         '-m', '--metadata STRING',
@@ -137,7 +137,7 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
   end
 
   def sanitize_cli
-    cli.ensure_par(taxon: '-T') unless cli[:reference]
+    cli.ensure_par(taxon: '-T')
     tasks = %w[reference complete chromosome scaffold contig]
     unless tasks.any? { |i| cli[i.to_sym] }
       raise 'No action requested: pick at least one type of genome'
@@ -204,7 +204,7 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
         'from(GenomeAssemblies).' \
         'usingschema(/schema/GenomeAssemblies).' \
         'matching(tab==["Prokaryotes"] and q=="' \
-          "#{cli[:taxon].tr('"', "'")}\"",
+          "#{cli[:taxon]&.tr('"', "'")}\"",
       fields: 'organism|organism,assembly|assembly,replicons|replicons,' \
         'level|level,ftp_path_genbank|ftp_path_genbank,' \
         'release_date|release_date,strain|strain',

data/lib/miga/cli/action/tax_dist.rb

@@ -99,13 +99,13 @@ class MiGA::Cli::Action::TaxDist < MiGA::Cli::Action
       ds_name = []
       File.open(tab, 'r') do |fh|
         fh.each_line do |ln|
-          if ln =~ /^ {#{(rank_i - 1) * 2}}\S+:\S+:/
+          if ln =~ /^ {0,#{(rank_i - 1) * 2}}\S+:\S+:/
             in_rank = nil
             ds_name = []
           elsif ln =~ /^ {#{rank_i * 2}}(#{rank}:(\S+)):/
             in_rank = $2 == '?' ? nil : $1
             ds_name = []
-          elsif ln =~ /^ *# (\S+)/ and not in_rank.nil?
+          elsif ln =~ /^ *# (\S+)/ && !in_rank.nil?
             ds_i = $1
             ds_name << ds_i
             ds_name.each do |ds_j|

data/lib/miga/cli/action/wf.rb

@@ -81,21 +81,22 @@ module MiGA::Cli::Action::Wf
       cli[:aai_p] = 'blast+'
       cli[:ani_p] = 'blast+'
     end
-    opt.on('--fast', 'Alias to: --aai-p diamond --ani-p fastani') do
+    opt.on('--fast', 'Alias to: --aai-p diamond --ani-p fastani (default)') do
       cli[:aai_p] = 'diamond'
       cli[:ani_p] = 'fastani'
     end
     opt.on(
       '--haai-p STRING',
-      'hAAI search engine. One of: blast+ (default), fastaai, blat, diamond, no'
+      'hAAI search engine. One of: blast+, fastaai, blat, diamond, fastaai, no',
+      'The default is "no" for clade projects and "fastaai" otherwise'
     ) { |v| cli[:haai_p] = v }
     opt.on(
       '--aai-p STRING',
-      'AAI search engine. One of: blast+ (default), blat, diamond'
+      'AAI search engine. One of: blast+, blat, diamond (default)'
     ) { |v| cli[:aai_p] = v }
     opt.on(
       '--ani-p STRING',
-      'ANI search engine. One of: blast+ (default), blat, fastani'
+      'ANI search engine. One of: blast+, blat, fastani (default)'
     ) { |v| cli[:ani_p] = v }
   end
 

data/lib/miga/daemon.rb

@@ -73,10 +73,10 @@ class MiGA::Daemon < MiGA::MiGA
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
     miga_say "Saving log to: #{output_file}" unless show_log?
-    queue_maintenance
-    load_status
     say 'Configuration options:'
     say @runopts.to_s
+    load_status
+    queue_maintenance(true)
   end
 
   ##
@@ -87,6 +87,7 @@ class MiGA::Daemon < MiGA::MiGA
     check_datasets or check_project
     if shutdown_when_done? && (jobs_running.size + jobs_to_run.size).zero?
       say 'Nothing else to do, shutting down'
+      exit_cleanup
       return false
     end
     flush!
@@ -102,13 +103,19 @@ class MiGA::Daemon < MiGA::MiGA
 
   ##
   # Queue maintenance tasks as an analysis job
-  def queue_maintenance
-    return if bypass_maintenance? || shutdown_when_done?
+  def queue_maintenance(force = false)
+    return if bypass_maintenance? || (!force && shutdown_when_done?)
 
     say 'Queueing maintenance tasks'
     queue_job(:maintenance)
   end
 
+  ##
+  # Remove temporary files on completion
+  def exit_cleanup
+    FileUtils.rm_f(File.join(daemon_home, 'status.json'))
+  end
+
   ##
   # Send +msg+ to +say+ as long as +level+ is at most +verbosity+
   def l_say(level, *msg)

data/lib/miga/dataset/result.rb

@@ -181,26 +181,30 @@ module MiGA::Dataset::Result
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       if result_files_exist?(base, '.2.clipped.fastq')
-        {
-          pair1: '.1.clipped.fastq',
-          pair2: '.2.clipped.fastq',
-          single: '.1.clipped.single.fastq'
-        }
+        { pair1: '.1.clipped.fastq', pair2: '.2.clipped.fastq' }
       else
         { single: '.1.clipped.fastq' }
       end
     ).tap do |r|
+      # Legacy files
       r.add_file(:trimming_sumary, "#{name}.1.fastq.trimmed.summary.txt")
+      r.add_file(:single, "#{name}.1.clipped.single.fastq")
     end
   end
 
   ##
   # Add result type +:read_quality+ at +base+ (no +_opts+ supported)
   def add_result_read_quality(base, _opts)
-    return nil unless result_files_exist?(base, %w[.solexaqa .fastqc])
+    return nil unless
+      result_files_exist?(base, %w[.post.1.html]) ||
+      result_files_exist?(base, %w[.solexaqa .fastqc])
 
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
+      pre_qc_1: '.pre.1.html', pre_qc_2: '.pre.2.html',
+      post_qc_1: '.post.1.html', post_qc_2: '.post.2.html',
+      adapter_detection: '.adapters.txt',
+      # Legacy files
       solexaqa: '.solexaqa', fastqc: '.fastqc'
     )
   end

data/lib/miga/json.rb

@@ -1,5 +1,4 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
 
 require 'json'
 

data/lib/miga/metadata.rb

@@ -105,7 +105,11 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Return the value of +k+ in #data
   def [](k)
-    data[k.to_sym]
+    if k.to_s =~ /(.+):(.+)/
+      data[$1.to_sym]&.fetch($2)
+    else
+      data[k.to_sym]
+    end
   end
 
   ##

data/lib/miga/parallel.rb

@@ -19,13 +19,18 @@ class MiGA::Parallel < MiGA::MiGA
     # 1. Unitary object from +enum+
     # 2. Index of the unitary object
     # 3. Index of the acting thread
-    def distribute(enum, threads)
-      process(threads) do |thr|
-        enum.each_with_index do |obj, idx|
-          yield(obj, idx, thr) if idx % threads == thr
-        end
+    def distribute(enum, threads, &blk)
+      process(threads) { |thr| thread_enum(enum, threads, thr, &blk) }
+    end
+
+    ##
+    # Enum through +enum+ executing the passed block only for thread with index
+    # +thr+, one of +threads+ threads. The passed block has the same arguments
+    # as the one in +#distribute+
+    def thread_enum(enum, threads, thr)
+      enum.each_with_index do |obj, idx|
+        yield(obj, idx, thr) if idx % threads == thr
       end
     end
   end
 end
-

data/lib/miga/project.rb

@@ -42,18 +42,18 @@ class MiGA::Project < MiGA::MiGA
   # Create an empty project
   def create
     unless MiGA::MiGA.initialized?
-      raise 'Impossible to create project in uninitialized MiGA.'
+      warn 'Projects cannot be processed yet, first run: miga init'
     end
 
-    dirs = [path] + @@FOLDERS.map { |d| "#{path}/#{d}" } +
-           @@DATA_FOLDERS.map { |d| "#{path}/data/#{d}" }
-    dirs.each { |d| Dir.mkdir(d) unless Dir.exist? d }
+    dirs = @@FOLDERS.map { |d| File.join(path, d) }
+    dirs += @@DATA_FOLDERS.map { |d| File.join(path, 'data', d) }
+    dirs.each { |d| FileUtils.mkdir_p(d) }
     @metadata = MiGA::Metadata.new(
-      File.expand_path('miga.project.json', path),
-      { datasets: [], name: File.basename(path) }
+      File.join(path, 'miga.project.json'),
+      datasets: [], name: File.basename(path)
     )
-    d_path = File.expand_path('daemon/daemon.json', path)
-    File.open(d_path, 'w') { |fh| fh.puts '{}' } unless File.exist? d_path
+    d_path = File.join(path, 'daemon', 'daemon.json')
+    File.open(d_path, 'w') { |fh| fh.puts '{}' } unless File.exist?(d_path)
     pull_hook :on_create
     self.load
   end

data/lib/miga/project/base.rb

@@ -131,15 +131,15 @@ module MiGA::Project::Base
     },
     haai_p: {
       desc: 'Value of aai.rb -p on hAAI', type: String,
-      default: proc { |project| project.clade? ? 'no' : 'blast+' },
-      in: %w[fastaai blast+ blast blat diamond no]
+      default: proc { |project| project.clade? ? 'no' : 'fastaai' },
+      in: %w[blast+ blast blat diamond fastaai no]
     },
     aai_p: {
-      desc: 'Value of aai.rb -p on AAI', default: 'blast+', type: String,
+      desc: 'Value of aai.rb -p on AAI', default: 'diamond', type: String,
       in: %w[blast+ blast blat diamond]
     },
     ani_p: {
-      desc: 'Value of ani.rb -p on ANI', default: 'blast+', type: String,
+      desc: 'Value of ani.rb -p on ANI', default: 'fastani', type: String,
       in: %w[blast+ blast blat fastani]
     },
     max_try: {