miga-base 0.7.15.1 → 0.7.16.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23ef43a6cf94c31d470263746a242d8b065899d11e694e4c281c0ec16fc4a335
-  data.tar.gz: 7b086c3b966827f7e49559409f89a4ec39af6bc2aec0164be8e06466bd765029
+  metadata.gz: f30e69f1e7031e63d0aeda691fffb2fa0e1bab46a8c859dfbc0212f6389b602d
+  data.tar.gz: 82a0acb779a387fc0f3722168526e200644db132bf906cbb749398412e83b152
 SHA512:
-  metadata.gz: 41e99ff68112b6f5d882b43a40e5ee3797e031f55d55b681ab5a9d56a83d70fe650813b3cd9127e35ac60cb88a56399b4165dd116395fcbb709c6a4748e794ef
-  data.tar.gz: '0870d670b3a90be77a39d027b00fdc83c01c91b9ab2e0e27ece54b1e90b64f6deb97da206bc93a1c4b1e9c9878d263b62028f63a9dac72349ebe3213f3551cfc'
+  metadata.gz: 3aa15d5a2a4340274b1c6f29fcfd7718ee825bdb200b06d4f7edbebf300e76f839d7a2c72a457ae3032ccd5fb3db1e9b0a9045e084e371c87f4a6bc86fdcc3b3
+  data.tar.gz: 37ba5fee0eef4c98e255f38a0afddf2ba7aee7f7ade45ba63550c7226967718b6aec781654adfd8fed7138f8723eba5b1d3ef20ffcb9c4b58c93a6df41b55f07

data/lib/miga/cli/action/doctor.rb

@@ -7,6 +7,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   include MiGA::Cli::Action::Doctor::Base
 
   def parse_cli
+    cli.defaults = { threads: 1 }
     cli.defaults = Hash[@@OPERATIONS.keys.map { |i| [i, true] }]
     cli.parse do |opt|
       operation_n = Hash[@@OPERATIONS.map { |k, v| [v[0], k] }]
@@ -24,6 +25,10 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
         @@OPERATIONS.each_key { |i| cli[i] = false }
         cli[op_k] = true
       end
+      opt.on(
+        '-t', '--threads INT', Integer,
+        "Concurrent threads to use. By default: #{cli[:threads]}"
+      ) { |v| cli[:threads] = v }
     end
   end
 
@@ -37,6 +42,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   @@OPERATIONS = {
     status: ['status', 'Update metadata status of all datasets'],
     db: ['databases', 'Check integrity of database files'],
+    bidir: ['bidirectional', 'Check distances are bidirectional'],
     dist: ['distances', 'Check distance summary tables'],
     files: ['files', 'Check for outdated files'],
     cds: ['cds', 'Check for gzipped genes and proteins'],
@@ -58,11 +64,19 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform status operation with MiGA::Cli +cli+
   def check_status(cli)
     cli.say 'Updating metadata status'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      d.recalculate_status
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        cli.load_project.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          d.recalculate_status if k % cli[:threads] == i
+        end
+      end
     end
+    Process.waitall
     cli.say
   end
 
@@ -70,18 +84,59 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform databases operation with MiGA::Cli +cli+
   def check_db(cli)
     cli.say 'Checking integrity of databases'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      each_database_file(d) do |db_file, metric, result|
-        check_sqlite3_database(db_file, metric) do
-          cli.say("  > Removing malformed database from #{d.name}:#{result}   ")
-          File.unlink(db_file)
-          r = d.result(result) or next
-          [r.path(:done), r.path].each { |f| File.unlink(f) if File.exist?(f) }
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        p.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
+          each_database_file(d) do |db_file, metric, result|
+            check_sqlite3_database(db_file, metric) do
+              cli.say(
+                "  > Removing malformed database from #{d.name}:#{result}   "
+              )
+              File.unlink(db_file)
+              r = d.result(result) or next
+              [r.path(:done), r.path].each do |f|
+                File.unlink(f) if File.exist?(f)
+              end
+            end
+          end
+        end
+      end
+    end
+    Process.waitall
+    cli.say
+  end
+
+  ##
+  # Perform bidirectional operation with MiGA::Cli +cli+
+  def check_bidir(cli)
+    cli.say 'Checking if reference distances are bidirectional'
+    ref_ds = cli.load_project.each_dataset.select(&:ref?)
+    ref_names = ref_ds.map(&:name)
+    n = ref_ds.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        ref_ds.each do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
+
+          saved = saved_targets(d)
+          next if saved.nil?
+
+          (ref_names - saved).each do |k|
+            save_bidirectional(cli.load_project.dataset(k), d)
+          end
         end
       end
     end
+    Process.waitall
     cli.say
   end
 

data/lib/miga/cli/action/doctor/base.rb

@@ -99,4 +99,55 @@ module MiGA::Cli::Action::Doctor::Base
     cli.say '- Removing tables, recompute'
     res.remove!
   end
+
+  ##
+  # Returns all targets identified by AAI
+  def saved_targets(dataset)
+    # Return nil if distance or database are not retrievable
+    dist = dataset.result(:distances) or return
+    path = dist.file_path(:aai_db) or return
+
+    o = []
+    SQLite3::Database.new(path) do |conn|
+      o = conn.execute('select seq2 from aai').map(&:first)
+    end
+    o
+  end
+
+  ##
+  # Saves all the distance estimates in +a+ -> +b+ into the +b+ databases
+  # (as +b+ -> +a+), where both +a+ and +b+ are MiGA::Dataset objects
+  def save_bidirectional(a, b)
+    each_database_file(a) do |db_file, metric, result|
+      data = nil
+      SQLite3::Database.new(db_file) do |conn|
+        data =
+          conn.execute(
+            "select seq1, seq2, #{metric}, sd, n, omega " +
+            "from #{metric} where seq2 = ? limit 1", b.name
+          ).first
+      end
+      next if data.nil? || data.empty?
+
+      db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
+      next unless File.exist?(db_file_b)
+
+      data[0], data[1] = data[1], data[0]
+      SQLite3::Database.new(db_file_b) do |conn|
+        attempts = 0
+        begin
+          attempts += 1
+          conn.execute(
+            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+            "values(?, ?, ?, ?, ?, ?)", data
+          )
+        rescue SQLite3::BusyException => e
+          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
+
+          sleep(1)
+          retry
+        end
+      end
+    end
+  end
 end

data/lib/miga/cli/action/get.rb

@@ -70,7 +70,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob = get_sub_cli
     p = cli.load_project
     glob.each do |sub_cli|
-      rd = create_remote_dataset(sub_cli)
+      rd = create_remote_dataset(sub_cli, p)
       next if rd.nil?
 
       if sub_cli[:get_md]
@@ -115,7 +115,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob
   end
 
-  def create_remote_dataset(sub_cli)
+  def create_remote_dataset(sub_cli, p)
     sub_cli.ensure_par(dataset: '-D', ids: '-I')
     unless sub_cli[:api_key].nil?
       ENV["#{sub_cli[:universe].to_s.upcase}_API_KEY"] = sub_cli[:api_key]

data/lib/miga/common.rb

@@ -52,10 +52,44 @@ class MiGA::MiGA
   # 1,000 otherwise.
   # The report goes to $stderr iff --verborse
   def advance(step, n = 0, total = nil, bin = true)
-    adv = total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-      ('%.1f%% (%s/%s)' % [100.0 * n / total,
-                           num_suffix(n, bin), num_suffix(total, bin)])
-    $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
+    # Initialize advance timing
+    @_advance_time ||= { last: nil, n: 0, avg: nil }
+    if n <= 1 || @_advance_time[:n] > n
+      @_advance_time[:last] = nil
+      @_advance_time[:n] = 0
+      @_advance_time[:avg]  = nil
+    end
+
+    # Estimate timing
+    adv_n = n - @_advance_time[:n]
+    unless total.nil? || @_advance_time[:last].nil? || adv_n <= 0
+      if adv_n.to_f/n > 0.001
+        this_time = Time.now - @_advance_time[:last]
+        this_avg = this_time / adv_n
+        @_advance_time[:avg] ||= this_avg
+        @_advance_time[:avg] = 0.9 * @_advance_time[:avg] + 0.1 * this_avg
+      end
+    end
+    @_advance_time[:last] = Time.now
+    @_advance_time[:n] = n
+
+    # Report
+    adv_vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
+    adv =
+      total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
+        ('%.1f%% (%s/%s)' % adv_vals)
+    left =
+      if @_advance_time[:avg].nil?
+        ''
+      else
+        left_time = @_advance_time[:avg] * (total - n) / 60 # <- in minutes
+        left_time < 0.01 ? '         ' :
+          left_time < 1 ? ('%.0fs left' % (left_time * 60)) :
+          left_time > 1440 ? ('%.1fd left' % (left_time / 1440)) :
+          left_time > 60 ? ('%.1fh left' % (left_time / 60)) :
+          ('%.1fm left' % left_time)
+      end
+    $stderr.print("[%s] %s %s %s    \r" % [Time.now, step, adv, left])
   end
 
   ##

data/lib/miga/daemon/base.rb

@@ -77,8 +77,9 @@ module MiGA::Daemon::Base
   ##
   # Writing file handler (IO) to the log file
   def logfh
+    @logfh ||= nil
     return $stderr if show_log?
-    return @logfh if @logfh && @logfh.closed?
+    return @logfh if @logfh && !@logfh.closed?
 
     @logfh = File.open(output_file, 'w')
   end

data/lib/miga/json.rb

@@ -34,17 +34,29 @@ class MiGA::Json < MiGA::MiGA
     # +opts+.
     def parse(path, opts = {})
       opts = default_opts(opts)
-      cont = opts[:contents] ? path : File.read(path)
-      raise "Empty descriptor: #{opts[:contents] ? "''" : path}." if cont.empty?
 
-      y = JSON.parse(cont,
-                     symbolize_names: opts[:symbolize],
-                     create_additions: opts[:additions])
+      # Read JSON
+      cont = path
+      12.times do
+        cont = File.read(path)
+        break unless cont.empty?
+        sleep 1 # Wait up to 12 seconds for racing processes (iff empty file)
+      end unless opts[:contents]
+      raise "Empty descriptor: #{opts[:contents] ? "''" : path}" if cont.empty?
+
+      # Parse JSON
+      params = { symbolize_names: opts[:symbolize],
+                 create_additions: opts[:additions] }
+      y = JSON.parse(cont, params)
+
+      # Add defaults
       unless opts[:default].nil?
         opts[:default] = parse(opts[:default]) if opts[:default].is_a? String
         y.each { |k, v| opts[:default][k] = v }
         y = opts[:default]
       end
+
+      # Return
       y
     end
 

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 15, 1]
+  VERSION = [0.7, 16, 4]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 8, 12)
+  VERSION_DATE = Date.new(2020, 10, 28)
 
   ##
   # Reference of MiGA.

data/scripts/project_stats.bash

@@ -12,7 +12,7 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
-miga doctor -P "$PROJECT" -v
+miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.15.1
+  version: 0.7.16.4
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-12 00:00:00.000000000 Z
+date: 2020-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons