miga-base 0.7.26.2 → 0.7.26.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a4edaaac424eb59fe5956b98c74b6926bfd57b90fd80e0dff9556c5ec75aae5
-  data.tar.gz: 0d2bba4f71074ef9efbeb03482a2d860b3c56ecd6734e7e8aeb82a928c2eec7d
+  metadata.gz: 0f2c9f2013b77f3a3b1dff0fbb73e74dad0b836d17d4a2129110a6095ffc7d8a
+  data.tar.gz: 0d7a1c879970dc9ffae5bae8cbbb51f6ef7ef13e41373fcc7cd5cc93db841104
 SHA512:
-  metadata.gz: '0052068c9d5a055d2c13694f1bacf755a37aa294d6db76ea8ed10a1464ddd58f6a536513f8d3f8817cf2491b7f2b01825fcf891bea75e72d699b0d2b9b82804e'
-  data.tar.gz: bbef9fa36b8b8900a3c9de38b3e69daf23fddf2fea59d0899d2781edecd96d32c6f612945ed8f32c373ea669ee7ee02c9c0d9db5db8b13cd92c95c94601494ca
+  metadata.gz: 4da1c02b538370a585efc788a71cbb9fafb5b8383c2882f0ab01d1bdec22a9dac2878bb1fa3a028f78ff19d22ecf70e50c67e20e71dc8a0308437ca19a6368db
+  data.tar.gz: 4eb38dafd4eb0dbaf69eed6fa734fec5c0d104d97d1f035cafdcd77365f93ecc723ac01eda643986f0676c4b4156a2504f92c1d98a02d0bd3955306e1a4924f5

data/lib/miga/cli/action/classify_wf.rb

@@ -50,7 +50,7 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
     # Input data
     ref_db = reference_db
     norun = %w[
-      project_stats haai_distances aai_distances ani_distances clade_finding
+      haai_distances aai_distances ani_distances clade_finding
     ]
     p_metadata = Hash[norun.map { |i| ["run_#{i}", false] }]
     p = create_project(
@@ -59,7 +59,7 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
       run_ssu: false, run_mytaxa_scan: false, run_distances: false
     )
     p.set_option(:ref_project, ref_db.path)
-    p.set_option(:tax_pvalue, cli[:pvalue], true)
+    p.set_option(:tax_pvalue, cli[:pvalue])
     # Run
     run_daemon
     summarize(%w[cds assembly essential_genes]) if cli[:summaries]

data/lib/miga/cli/action/derep_wf.rb

@@ -52,7 +52,7 @@ class MiGA::Cli::Action::DerepWf < MiGA::Cli::Action
     # Input data
     p = create_project(
       :assembly,
-      { run_project_stats: false, run_clades: false },
+      { run_clades: false },
       { run_mytaxa_scan: false, run_ssu: false }
     )
     p.set_option(:gsp_metric, cli[:metric].to_s)

data/lib/miga/cli/action/doctor.rb

@@ -93,7 +93,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
           k += 1
           cli.advance('Datasets:', k, n, false) if i == 0
           next unless k % cli[:threads] == i
-          each_database_file(d) do |db_file, metric, result|
+          each_database_file(d) do |db_file, metric, result, _rank|
             check_sqlite3_database(db_file, metric) do
               cli.say(
                 "  > Removing malformed database from #{d.name}:#{result}   "
@@ -119,24 +119,36 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
     ref_ds = cli.load_project.each_dataset.select(&:ref?)
     ref_names = ref_ds.map(&:name)
     n = ref_ds.size
-    (0 .. cli[:threads] - 1).map do |i|
-      Process.fork do
-        k = 0
-        ref_ds.each do |d|
-          k += 1
-          cli.advance('Datasets:', k, n, false) if i == 0
-          next unless k % cli[:threads] == i
-
-          saved = saved_targets(d)
-          next if saved.nil?
 
-          (ref_names - saved).each do |k|
-            save_bidirectional(cli.load_project.dataset(k), d)
-          end
+    # Read data first (threaded)
+    @distances = { aai: {}, ani: {} }
+    Dir.mktmpdir do |tmp|
+      MiGA::Parallel.process(cli[:threads]) do |thr|
+        idx = 0
+        ref_ds.each do |ds|
+          cli.advance('Reading:', idx + 1, n, false) if thr == 0
+          read_bidirectional(ds) if idx % cli[:threads] == thr
+          idx += 1
+        end
+        File.open("#{tmp}/#{thr}.json", 'w') do |fh|
+          fh.print JSON.fast_generate(@distances)
         end
       end
+      cli.say
+
+      cli[:threads].times do |i|
+        cli.advance('Merging:', i + 1, cli[:threads], false)
+        o = MiGA::Json.parse("#{tmp}/#{i}.json", symbolize: false)
+        o.each { |k, v| @distances[k.to_sym].merge!(v) }
+      end
+      cli.say
+    end
+
+    # Write missing values (threaded)
+    MiGA::Parallel.distribute(ref_ds, cli[:threads]) do |ds, idx, thr|
+      cli.advance('Datasets:', idx + 1, n, false) if thr == 0
+      save_bidirectional(ds)
     end
-    Process.waitall
     cli.say
   end
 

data/lib/miga/cli/action/doctor/base.rb

@@ -16,22 +16,28 @@ module MiGA::Cli::Action::Doctor::Base
   end
 
   def each_database_file(dataset, &blk)
-    ref_db = { '01.haai' => :aai, '02.aai' => :aai, '03.ani' => :ani }
-    qry_db = { '.haai.db' => :aai, '.aai.db' => :aai, '.ani.db' => :ani }
+    ref_db = {
+      haai: ['01.haai', :aai], aai: ['02.aai', :aai], ani: ['03.ani', :ani]
+    }
+    qry_db = {
+      haai: ['.haai.db', :aai], aai: ['.aai.db', :aai], ani: ['.ani.db', :ani]
+    }
     base = File.join(dataset.project.path, 'data', '09.distances')
     result = :distances
     if dataset.ref?
       file_db = "#{dataset.name}.db"
-      ref_db.each do |dir, metric|
+      ref_db.each do |rank, v|
+        dir, metric = *v
         file = File.join(base, dir, file_db)
-        blk[file, metric, result] if File.exist? file
+        blk[file, metric, result, rank] if File.exist? file
       end
       base = File.join(base, '05.taxonomy')
       result = :taxonomy
     end
-    qry_db.each do |ext, metric|
+    qry_db.each do |rank, v|
+      ext, metric = *v
       file = File.join(base, "#{dataset.name}#{ext}")
-      blk[file, metric, result] if File.exist? file
+      blk[file, metric, result, rank] if File.exist? file
     end
   end
 
@@ -109,25 +115,41 @@ module MiGA::Cli::Action::Doctor::Base
   end
 
   ##
-  # Saves all the distance estimates in +a+ -> +b+ into the +b+ databases
-  # (as +b+ -> +a+), where both +a+ and +b+ are MiGA::Dataset objects
-  def save_bidirectional(a, b)
-    each_database_file(a) do |db_file, metric, result|
-      data = nil
-      data = MiGA::SQLite.new(db_file).run(
-        "select seq1, seq2, #{metric}, sd, n, omega " +
-        "from #{metric} where seq2 = ? limit 1", b.name
-      ).first
+  # Reads all the distance estimates in +a+ -> *, and saves them in memory
+  # in the +@distances+ variable.
+  def read_bidirectional(a)
+    each_database_file(a) do |db_file, metric, result, rank|
+      next if rank == :haai # No need for hAAI to be bidirectional
+
+      sql = "select seq2, #{metric}, sd, n, omega from #{metric}"
+      data = MiGA::SQLite.new(db_file).run(sql)
       next if data.nil? || data.empty?
 
-      db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
-      next unless File.exist?(db_file_b)
+      @distances[rank][a.name] ||= {}
+      data.each { |row| @distances[rank][a.name][row.shift] = row }
+    end
+  end
 
-      data[0], data[1] = data[1], data[0]
-      MiGA::SQLite.new(db_file_b).run(
-        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-        "values(?, ?, ?, ?, ?, ?)", data
-      )
+  ##
+  # Saves all the distance estimates in * -> +a+ into the +a+ databases
+  # (as +a+ -> *), where +a+ is a MiGA::Dataset object
+  def save_bidirectional(a)
+    each_database_file(a) do |db_file, metric, result, rank|
+      next if rank == :haai # No need for hAAI to be bidirectional
+
+      b2a = @distances[rank].map { |b_name, v| b_name if v[a.name] }.compact
+      a2b = @distances[rank][a.name].keys
+      SQLite3::Database.new(db_file) do |db|
+        sql = <<~SQL
+          insert into #{metric}(seq1, seq2, #{metric}, sd, n, omega) \
+          values(?, ?, ?, ?, ?, ?);
+        SQL
+        db.execute('BEGIN TRANSACTION;')
+        (b2a - a2b).each do |b_name|
+          db.execute(sql, [a.name, b_name] + @distances[rank][b_name][a.name])
+        end
+        db.execute('COMMIT;')
+      end
     end
   end
 end

data/lib/miga/common.rb

@@ -3,6 +3,7 @@
 
 require 'miga/version'
 require 'miga/json'
+require 'miga/parallel'
 require 'miga/common/base'
 require 'miga/common/path'
 require 'miga/common/format'

data/lib/miga/json.rb

@@ -45,8 +45,10 @@ class MiGA::Json < MiGA::MiGA
       raise "Empty descriptor: #{opts[:contents] ? "''" : path}" if cont.empty?
 
       # Parse JSON
-      params = { symbolize_names: opts[:symbolize],
-                 create_additions: opts[:additions] }
+      params = {
+        symbolize_names: opts[:symbolize],
+        create_additions: opts[:additions]
+      }
       y = JSON.parse(cont, params)
 
       # Add defaults

data/lib/miga/parallel.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+##
+# Parallel execution in MiGA.
+class MiGA::Parallel < MiGA::MiGA
+  class << self
+    ##
+    # Executes the passed block with the thread number as argument (0-numbered)
+    # in +threads+ processes
+    def process(threads)
+      threads.times do |i|
+        Process.fork { yield(i) }
+      end
+      Process.waitall
+    end
+
+    ##
+    # Distributes +enum+ across +threads+ and calls the passed block with args:
+    # 1. Unitary object from +enum+
+    # 2. Index of the unitary object
+    # 3. Index of the acting thread
+    def distribute(enum, threads)
+      process(threads) do |thr|
+        enum.each_with_index do |obj, idx|
+          yield(obj, idx, thr) if idx % threads == thr
+        end
+      end
+    end
+  end
+end
+

data/lib/miga/sqlite.rb

@@ -37,8 +37,9 @@ class MiGA::SQLite < MiGA::MiGA
   # Executes +cmd+ and returns the result
   def run(*cmd)
     busy_attempts ||= 0
-    conn = SQLite3::Database.new(path)
-    conn.execute(*cmd)
+    y = nil
+    SQLite3::Database.new(path) { |conn| y = conn.execute(*cmd) }
+    y
   rescue SQLite3::BusyException => e
     busy_attempts += 1
     raise "Database busy #{path}: #{e.message}" if busy_attempts >= 3

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 26, 2].freeze
+  VERSION = [0.7, 26, 3].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 3, 1)
+  VERSION_DATE = Date.new(2021, 3, 11)
 
   ##
   # Reference of MiGA.

data/test/remote_dataset_test.rb

@@ -47,7 +47,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
   end
 
   def test_net_ftp
-    cjac = 'ftp://ftp.ebi.ac.uk/pub/databases/ena/tsa/public/ga/GAPJ01.fasta.gz'
+    cjac = 'ftp://ftp.ebi.ac.uk/pub/databases/ena/tsa/public/gap/GAPJ01.fasta.gz'
     n = 'Cjac_L14'
     rd = MiGA::RemoteDataset.new(cjac, :assembly_gz, :web)
     assert_equal([cjac], rd.ids)

data/utils/distance/database.rb

@@ -126,7 +126,6 @@ module MiGA::DistanceRunner::Database
   def batch_data_to_db(metric, data)
     db = tmp_dbs[metric]
     table = metric == :haai ? :aai : metric
-    `cp #{db} ~/here.db`
     SQLite3::Database.new(db) do |conn|
       data.each do |k, v|
         sql = <<~SQL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.26.2
+  version: 0.7.26.3
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-01 00:00:00.000000000 Z
+date: 2021-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -187,6 +187,7 @@ files:
 - lib/miga/json.rb
 - lib/miga/lair.rb
 - lib/miga/metadata.rb
+- lib/miga/parallel.rb
 - lib/miga/project.rb
 - lib/miga/project/base.rb
 - lib/miga/project/dataset.rb