miga-base 0.7.25.2 → 0.7.26.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45e7273aec3b6dae9f942561ee39149fa2c50930d1f3e515666e3c2a4434e7c1
-  data.tar.gz: a2eaa3b270cd2f171b47a799841957ef2b53b3055138ca6e175615a32e969cd1
+  metadata.gz: 0f2c9f2013b77f3a3b1dff0fbb73e74dad0b836d17d4a2129110a6095ffc7d8a
+  data.tar.gz: 0d7a1c879970dc9ffae5bae8cbbb51f6ef7ef13e41373fcc7cd5cc93db841104
 SHA512:
-  metadata.gz: ec0538b54ad490ddb57deb7d4cb1b4dcbcdedc7c5cfd6fb5afcf40ac1c31154f9b750c72ee0c8aab08991d77d4e275bbeccc8f49c8c65043ae304f99d0211fe6
-  data.tar.gz: d992f9dc1c52925b88d9132838d9ec27906815a656362346b294fdef2edec669a9f0a161c635cbdf2a1c6cf8ac2f8918179db0b1a0d431e445180eb2033c1650
+  metadata.gz: 4da1c02b538370a585efc788a71cbb9fafb5b8383c2882f0ab01d1bdec22a9dac2878bb1fa3a028f78ff19d22ecf70e50c67e20e71dc8a0308437ca19a6368db
+  data.tar.gz: 4eb38dafd4eb0dbaf69eed6fa734fec5c0d104d97d1f035cafdcd77365f93ecc723ac01eda643986f0676c4b4156a2504f92c1d98a02d0bd3955306e1a4924f5

data/lib/miga/cli/action/classify_wf.rb

@@ -50,7 +50,7 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
     # Input data
     ref_db = reference_db
     norun = %w[
-      project_stats haai_distances aai_distances ani_distances clade_finding
+      haai_distances aai_distances ani_distances clade_finding
     ]
     p_metadata = Hash[norun.map { |i| ["run_#{i}", false] }]
     p = create_project(
@@ -59,7 +59,7 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
       run_ssu: false, run_mytaxa_scan: false, run_distances: false
     )
     p.set_option(:ref_project, ref_db.path)
-    p.set_option(:tax_pvalue, cli[:pvalue], true)
+    p.set_option(:tax_pvalue, cli[:pvalue])
     # Run
     run_daemon
     summarize(%w[cds assembly essential_genes]) if cli[:summaries]

data/lib/miga/cli/action/derep_wf.rb

@@ -52,7 +52,7 @@ class MiGA::Cli::Action::DerepWf < MiGA::Cli::Action
     # Input data
     p = create_project(
       :assembly,
-      { run_project_stats: false, run_clades: false },
+      { run_clades: false },
       { run_mytaxa_scan: false, run_ssu: false }
     )
     p.set_option(:gsp_metric, cli[:metric].to_s)

data/lib/miga/cli/action/doctor.rb

@@ -93,7 +93,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
           k += 1
           cli.advance('Datasets:', k, n, false) if i == 0
           next unless k % cli[:threads] == i
-          each_database_file(d) do |db_file, metric, result|
+          each_database_file(d) do |db_file, metric, result, _rank|
             check_sqlite3_database(db_file, metric) do
               cli.say(
                 "  > Removing malformed database from #{d.name}:#{result}   "
@@ -119,24 +119,36 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
     ref_ds = cli.load_project.each_dataset.select(&:ref?)
     ref_names = ref_ds.map(&:name)
     n = ref_ds.size
-    (0 .. cli[:threads] - 1).map do |i|
-      Process.fork do
-        k = 0
-        ref_ds.each do |d|
-          k += 1
-          cli.advance('Datasets:', k, n, false) if i == 0
-          next unless k % cli[:threads] == i
-
-          saved = saved_targets(d)
-          next if saved.nil?
 
-          (ref_names - saved).each do |k|
-            save_bidirectional(cli.load_project.dataset(k), d)
-          end
+    # Read data first (threaded)
+    @distances = { aai: {}, ani: {} }
+    Dir.mktmpdir do |tmp|
+      MiGA::Parallel.process(cli[:threads]) do |thr|
+        idx = 0
+        ref_ds.each do |ds|
+          cli.advance('Reading:', idx + 1, n, false) if thr == 0
+          read_bidirectional(ds) if idx % cli[:threads] == thr
+          idx += 1
+        end
+        File.open("#{tmp}/#{thr}.json", 'w') do |fh|
+          fh.print JSON.fast_generate(@distances)
         end
       end
+      cli.say
+
+      cli[:threads].times do |i|
+        cli.advance('Merging:', i + 1, cli[:threads], false)
+        o = MiGA::Json.parse("#{tmp}/#{i}.json", symbolize: false)
+        o.each { |k, v| @distances[k.to_sym].merge!(v) }
+      end
+      cli.say
+    end
+
+    # Write missing values (threaded)
+    MiGA::Parallel.distribute(ref_ds, cli[:threads]) do |ds, idx, thr|
+      cli.advance('Datasets:', idx + 1, n, false) if thr == 0
+      save_bidirectional(ds)
     end
-    Process.waitall
     cli.say
   end
 

data/lib/miga/cli/action/doctor/base.rb

@@ -16,22 +16,28 @@ module MiGA::Cli::Action::Doctor::Base
   end
 
   def each_database_file(dataset, &blk)
-    ref_db = { '01.haai' => :aai, '02.aai' => :aai, '03.ani' => :ani }
-    qry_db = { '.haai.db' => :aai, '.aai.db' => :aai, '.ani.db' => :ani }
+    ref_db = {
+      haai: ['01.haai', :aai], aai: ['02.aai', :aai], ani: ['03.ani', :ani]
+    }
+    qry_db = {
+      haai: ['.haai.db', :aai], aai: ['.aai.db', :aai], ani: ['.ani.db', :ani]
+    }
     base = File.join(dataset.project.path, 'data', '09.distances')
     result = :distances
     if dataset.ref?
       file_db = "#{dataset.name}.db"
-      ref_db.each do |dir, metric|
+      ref_db.each do |rank, v|
+        dir, metric = *v
         file = File.join(base, dir, file_db)
-        blk[file, metric, result] if File.exist? file
+        blk[file, metric, result, rank] if File.exist? file
       end
       base = File.join(base, '05.taxonomy')
       result = :taxonomy
     end
-    qry_db.each do |ext, metric|
+    qry_db.each do |rank, v|
+      ext, metric = *v
       file = File.join(base, "#{dataset.name}#{ext}")
-      blk[file, metric, result] if File.exist? file
+      blk[file, metric, result, rank] if File.exist? file
     end
   end
 
@@ -109,25 +115,41 @@ module MiGA::Cli::Action::Doctor::Base
   end
 
   ##
-  # Saves all the distance estimates in +a+ -> +b+ into the +b+ databases
-  # (as +b+ -> +a+), where both +a+ and +b+ are MiGA::Dataset objects
-  def save_bidirectional(a, b)
-    each_database_file(a) do |db_file, metric, result|
-      data = nil
-      data = MiGA::SQLite.new(db_file).run(
-        "select seq1, seq2, #{metric}, sd, n, omega " +
-        "from #{metric} where seq2 = ? limit 1", b.name
-      ).first
+  # Reads all the distance estimates in +a+ -> *, and saves them in memory
+  # in the +@distances+ variable.
+  def read_bidirectional(a)
+    each_database_file(a) do |db_file, metric, result, rank|
+      next if rank == :haai # No need for hAAI to be bidirectional
+
+      sql = "select seq2, #{metric}, sd, n, omega from #{metric}"
+      data = MiGA::SQLite.new(db_file).run(sql)
       next if data.nil? || data.empty?
 
-      db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
-      next unless File.exist?(db_file_b)
+      @distances[rank][a.name] ||= {}
+      data.each { |row| @distances[rank][a.name][row.shift] = row }
+    end
+  end
 
-      data[0], data[1] = data[1], data[0]
-      MiGA::SQLite.new(db_file_b).run(
-        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-        "values(?, ?, ?, ?, ?, ?)", data
-      )
+  ##
+  # Saves all the distance estimates in * -> +a+ into the +a+ databases
+  # (as +a+ -> *), where +a+ is a MiGA::Dataset object
+  def save_bidirectional(a)
+    each_database_file(a) do |db_file, metric, result, rank|
+      next if rank == :haai # No need for hAAI to be bidirectional
+
+      b2a = @distances[rank].map { |b_name, v| b_name if v[a.name] }.compact
+      a2b = @distances[rank][a.name].keys
+      SQLite3::Database.new(db_file) do |db|
+        sql = <<~SQL
+          insert into #{metric}(seq1, seq2, #{metric}, sd, n, omega) \
+          values(?, ?, ?, ?, ?, ?);
+        SQL
+        db.execute('BEGIN TRANSACTION;')
+        (b2a - a2b).each do |b_name|
+          db.execute(sql, [a.name, b_name] + @distances[rank][b_name][a.name])
+        end
+        db.execute('COMMIT;')
+      end
     end
   end
 end

data/lib/miga/cli/action/init.rb

@@ -14,6 +14,8 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     cli.defaults = {
       mytaxa: nil,
       rdp: nil,
+      reads: nil,
+      optional: nil,
       config: File.join(ENV['MIGA_HOME'], '.miga_modules'),
       ask: false,
       auto: false,
@@ -35,6 +37,16 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
         'Should I try setting up the RDP classifier?',
         'By default: interactive (true if --auto)'
       ) { |v| cli[:rdp] = v }
+      opt.on(
+        '--[no-]read-processing',
+        'Should I try setting up read processing software?',
+        'By default: interactive (true if --auto)'
+      ) { |v| cli[:reads] = v }
+      opt.on(
+        '--[no-]optional',
+        'Should I try setting up the optional software?',
+        'Automatically sets answers for mytaxa, rdp, and reads'
+      ) { |v| cli[:optional] = v }
       opt.on(
         '--daemon-type STRING',
         'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
@@ -104,6 +116,8 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
     rc_fh.puts "export MIGA_MYTAXA='#{cli[:mytaxa] ? 'yes' : 'no'}'"
     ask_for_optional(:rdp, 'RDP classifier')
     rc_fh.puts "export MIGA_RDP='#{cli[:rdp] ? 'yes' : 'no'}'"
+    ask_for_optional(:reads, 'read processing')
+    rc_fh.puts "export MIGA_READS='#{cli[:reads] ? 'yes' : 'no'}'"
     paths = {}
     rc_fh.puts 'MIGA_PATH=""'
     req_path = File.expand_path('utils/requirements.txt', MiGA.root_path)
@@ -123,20 +137,20 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
   def define_software(ln)
     r = ln.chomp.split(/\t+/)
     return if %w[Software --------].include?(r[0])
-    return if r[0] =~ /\(mytaxa\)$/ && !cli[:mytaxa]
-    return if r[0] =~ /\(rdp\)$/ && !cli[:rdp]
+    %i[mytaxa rdp reads].each { |i| return if r[0] =~ /\(#{i}\)$/ && !cli[i] }
 
     r
   end
 
   def ask_for_optional(symbol, name)
-    if cli[symbol].nil?
-      cli[symbol] =
-        cli.ask_user(
-          "Should I include #{name} modules?",
-          'yes', %w(yes no)
-        ) == 'yes'
-    end
+    cli[symbol] = cli[:optional] if !cli[:optional].nil? && cli[symbol].nil?
+    return cli[symbol] unless cli[symbol].nil?
+
+    cli[symbol] =
+      cli.ask_user(
+        "Should I include #{name} modules?",
+        'yes', %w(yes no)
+      ) == 'yes'
   end
 
   def find_software(exec)

data/lib/miga/common.rb

@@ -3,6 +3,7 @@
 
 require 'miga/version'
 require 'miga/json'
+require 'miga/parallel'
 require 'miga/common/base'
 require 'miga/common/path'
 require 'miga/common/format'

data/lib/miga/json.rb

@@ -45,8 +45,10 @@ class MiGA::Json < MiGA::MiGA
       raise "Empty descriptor: #{opts[:contents] ? "''" : path}" if cont.empty?
 
       # Parse JSON
-      params = { symbolize_names: opts[:symbolize],
-                 create_additions: opts[:additions] }
+      params = {
+        symbolize_names: opts[:symbolize],
+        create_additions: opts[:additions]
+      }
       y = JSON.parse(cont, params)
 
       # Add defaults

data/lib/miga/parallel.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+##
+# Parallel execution in MiGA.
+class MiGA::Parallel < MiGA::MiGA
+  class << self
+    ##
+    # Executes the passed block with the thread number as argument (0-numbered)
+    # in +threads+ processes
+    def process(threads)
+      threads.times do |i|
+        Process.fork { yield(i) }
+      end
+      Process.waitall
+    end
+
+    ##
+    # Distributes +enum+ across +threads+ and calls the passed block with args:
+    # 1. Unitary object from +enum+
+    # 2. Index of the unitary object
+    # 3. Index of the acting thread
+    def distribute(enum, threads)
+      process(threads) do |thr|
+        enum.each_with_index do |obj, idx|
+          yield(obj, idx, thr) if idx % threads == thr
+        end
+      end
+    end
+  end
+end
+

data/lib/miga/sqlite.rb

@@ -37,8 +37,9 @@ class MiGA::SQLite < MiGA::MiGA
   # Executes +cmd+ and returns the result
   def run(*cmd)
     busy_attempts ||= 0
-    conn = SQLite3::Database.new(path)
-    conn.execute(*cmd)
+    y = nil
+    SQLite3::Database.new(path) { |conn| y = conn.execute(*cmd) }
+    y
   rescue SQLite3::BusyException => e
     busy_attempts += 1
     raise "Database busy #{path}: #{e.message}" if busy_attempts >= 3

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 25, 2].freeze
+  VERSION = [0.7, 26, 3].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 2, 25)
+  VERSION_DATE = Date.new(2021, 3, 11)
 
   ##
   # Reference of MiGA.

data/test/remote_dataset_test.rb

@@ -47,7 +47,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
   end
 
   def test_net_ftp
-    cjac = 'ftp://ftp.ebi.ac.uk/pub/databases/ena/tsa/public/ga/GAPJ01.fasta.gz'
+    cjac = 'ftp://ftp.ebi.ac.uk/pub/databases/ena/tsa/public/gap/GAPJ01.fasta.gz'
     n = 'Cjac_L14'
     rd = MiGA::RemoteDataset.new(cjac, :assembly_gz, :web)
     assert_equal([cjac], rd.ids)

data/utils/distance/database.rb

@@ -126,7 +126,6 @@ module MiGA::DistanceRunner::Database
   def batch_data_to_db(metric, data)
     db = tmp_dbs[metric]
     table = metric == :haai ? :aai : metric
-    `cp #{db} ~/here.db`
     SQLite3::Database.new(db) do |conn|
       data.each do |k, v|
         sql = <<~SQL

data/utils/distance/runner.rb

@@ -102,7 +102,8 @@ class MiGA::DistanceRunner
     cl_path = res.file_path :clades_ani95
     if !cl_path.nil? && File.size?(cl_path) && tsk[0] == :clade_finding
       clades = File.foreach(cl_path).map { |i| i.chomp.split(',') }
-      sbj_datasets = clades.find { |i| i.include?(closest[:ds]) }
+      sbj_dataset_names = clades.find { |i| i.include?(closest[:ds]) }
+      sbj_datasets = sbj_dataset_names&.map { |i| ref_project.dataset(i) }
       ani_after_aai(sbj_datasets, 80.0) if sbj_datasets
     end
 

data/utils/requirements.txt

@@ -10,12 +10,12 @@ FastANI		fastANI		https://github.com/ParBLiSS/FastANI				Required version: 1.1+
 HMMer 3.0+	hmmsearch	http://hmmer.janelia.org/software
 Bedtools	bedtools	http://bedtools.readthedocs.org/en/latest/
 Prodigal	prodigal	http://prodigal.ornl.gov
-IDBA		idba_ud		http://i.cs.hku.hk/~alse/hkubrg/projects/idba
 MCL		mcl		http://micans.org/mcl/
 Barrnap		barrnap		http://www.vicbioinformatics.com/software.barrnap.shtml
-Scythe		scythe		https://github.com/vsbuffalo/scythe				Required version: 0.991+
-FastQC		fastqc		http://www.bioinformatics.babraham.ac.uk/projects/fastqc
-SolexaQA++	SolexaQA++	http://solexaqa.sourceforge.net					Required version: v3.1.3+
+IDBA (reads)	idba_ud		http://i.cs.hku.hk/~alse/hkubrg/projects/idba
+Scythe (reads)	scythe		https://github.com/vsbuffalo/scythe				Required version: 0.991+
+FastQC (reads)	fastqc		http://www.bioinformatics.babraham.ac.uk/projects/fastqc
+SolexaQA++ (reads)	SolexaQA++	http://solexaqa.sourceforge.net				Required version: v3.1.3+
 OpenJDK (rdp)	java		https://adoptopenjdk.net/					Any Java VM would work
 MyTaxa (mytaxa)	MyTaxa		http://enve-omics.ce.gatech.edu/mytaxa
 Krona (mytaxa)	ktImportText	https://github.com/marbl/Krona/wiki

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.25.2
+  version: 0.7.26.3
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-25 00:00:00.000000000 Z
+date: 2021-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -187,6 +187,7 @@ files:
 - lib/miga/json.rb
 - lib/miga/lair.rb
 - lib/miga/metadata.rb
+- lib/miga/parallel.rb
 - lib/miga/project.rb
 - lib/miga/project/base.rb
 - lib/miga/project/dataset.rb
@@ -554,7 +555,7 @@ homepage: http://enve-omics.ce.gatech.edu/miga
 licenses:
 - Artistic-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options:
 - lib
 - README.md
@@ -576,7 +577,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.4
-signing_key:
+signing_key: 
 specification_version: 4
 summary: MiGA
 test_files: []