miga-base 0.7.16.1 → 0.7.16.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c515f327c9d7178b1891283f849d422885b3640f8cded39cfccd048e354acf16
-  data.tar.gz: 889a399e0fa68b54c901ec513e2049ab046593bc433185511516dbf9b3cedb7d
+  metadata.gz: 51173aea1659d14d35f485b22b4371bdf8cf9d16ae81c673d09c25cf3e36d1ac
+  data.tar.gz: 4c6cbaed4bc9f0ea2f71d75ccfd22e3b0ac38e62c138166b768feb91c9a804b3
 SHA512:
-  metadata.gz: f4a9d5515ddf4b208b81612ceafcdae0eeb9cd37266df5f0175cef859adb9b24fd6e9ac5adf384ad9c91f8ecc08d6d5522eef8b9dd96e7d39740b14f83fdaf53
-  data.tar.gz: 117bc5b917e1008084e064cd4d87f290a83b6962e93560abfad254d23345a7685bf9ad3a345f2ade98ea5c13b4be52ccf608683ede8c320f3b8ee1250c2cec32
+  metadata.gz: 6e00443678373cb3125e1d73d111ff909cfaa092c92ffd00089e2d122bad0f298133e4c4b2da00370d4ff56e3d1281b8e3ebe316e23fa675b73b217b6045e4e8
+  data.tar.gz: f5d5f2981f86e0a79a1c96b81cdb76853eeacd11a14c23d630bcf067d390074d40273159b2b95c3ea33d88d1ca1c66bce19338809af410e48a0db2138d4974d3

data/lib/miga/cli/action/doctor.rb

@@ -7,6 +7,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   include MiGA::Cli::Action::Doctor::Base
 
   def parse_cli
+    cli.defaults = { threads: 1 }
     cli.defaults = Hash[@@OPERATIONS.keys.map { |i| [i, true] }]
     cli.parse do |opt|
       operation_n = Hash[@@OPERATIONS.map { |k, v| [v[0], k] }]
@@ -24,6 +25,10 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
         @@OPERATIONS.each_key { |i| cli[i] = false }
         cli[op_k] = true
       end
+      opt.on(
+        '-t', '--threads INT', Integer,
+        "Concurrent threads to use. By default: #{cli[:threads]}"
+      ) { |v| cli[:threads] = v }
     end
   end
 
@@ -59,11 +64,19 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform status operation with MiGA::Cli +cli+
   def check_status(cli)
     cli.say 'Updating metadata status'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      d.recalculate_status
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        cli.load_project.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          d.recalculate_status if k % cli[:threads] == i
+        end
+      end
     end
+    Process.waitall
     cli.say
   end
 
@@ -71,36 +84,59 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform databases operation with MiGA::Cli +cli+
   def check_db(cli)
     cli.say 'Checking integrity of databases'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      each_database_file(d) do |db_file, metric, result|
-        check_sqlite3_database(db_file, metric) do
-          cli.say("  > Removing malformed database from #{d.name}:#{result}   ")
-          File.unlink(db_file)
-          r = d.result(result) or next
-          [r.path(:done), r.path].each { |f| File.unlink(f) if File.exist?(f) }
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        p.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
+          each_database_file(d) do |db_file, metric, result|
+            check_sqlite3_database(db_file, metric) do
+              cli.say(
+                "  > Removing malformed database from #{d.name}:#{result}   "
+              )
+              File.unlink(db_file)
+              r = d.result(result) or next
+              [r.path(:done), r.path].each do |f|
+                File.unlink(f) if File.exist?(f)
+              end
+            end
+          end
         end
       end
     end
+    Process.waitall
     cli.say
   end
 
   ##
   # Perform bidirectional operation with MiGA::Cli +cli+
   def check_bidir(cli)
-    cli.say 'Checking that reference distances are bidirectional'
+    cli.say 'Checking if reference distances are bidirectional'
     ref_ds = cli.load_project.each_dataset.select(&:ref?)
     ref_names = ref_ds.map(&:name)
-    n, k = ref_ds.size, 0
-    ref_ds.each do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      saved = saved_targets(d)
-      next if saved.nil?
+    n = ref_ds.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        ref_ds.each do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
 
-      to_save = ref_names - saved
-      to_save.each { |k| save_bidirectional(cli.load_project.dataset(k), d) }
+          saved = saved_targets(d)
+          next if saved.nil?
+
+          (ref_names - saved).each do |k|
+            save_bidirectional(cli.load_project.dataset(k), d)
+          end
+        end
+      end
     end
+    Process.waitall
     cli.say
   end
 

data/lib/miga/cli/action/doctor/base.rb

@@ -134,10 +134,19 @@ module MiGA::Cli::Action::Doctor::Base
 
       data[0], data[1] = data[1], data[0]
       SQLite3::Database.new(db_file_b) do |conn|
-        conn.execute(
-          "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-          "values(?, ?, ?, ?, ?, ?)", data
-        )
+        attempts = 0
+        begin
+          attempts += 1
+          conn.execute(
+            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+            "values(?, ?, ?, ?, ?, ?)", data
+          )
+        rescue SQLite3::BusyException => e
+          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
+
+          sleep(1)
+          retry
+        end
       end
     end
   end

data/lib/miga/cli/action/get.rb

@@ -70,7 +70,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob = get_sub_cli
     p = cli.load_project
     glob.each do |sub_cli|
-      rd = create_remote_dataset(sub_cli)
+      rd = create_remote_dataset(sub_cli, p)
       next if rd.nil?
 
       if sub_cli[:get_md]
@@ -115,7 +115,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob
   end
 
-  def create_remote_dataset(sub_cli)
+  def create_remote_dataset(sub_cli, p)
     sub_cli.ensure_par(dataset: '-D', ids: '-I')
     unless sub_cli[:api_key].nil?
       ENV["#{sub_cli[:universe].to_s.upcase}_API_KEY"] = sub_cli[:api_key]

data/lib/miga/common.rb

@@ -52,10 +52,39 @@ class MiGA::MiGA
   # 1,000 otherwise.
   # The report goes to $stderr iff --verborse
   def advance(step, n = 0, total = nil, bin = true)
-    adv = total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-      ('%.1f%% (%s/%s)' % [100.0 * n / total,
-                           num_suffix(n, bin), num_suffix(total, bin)])
-    $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
+    # Initialize advance timing
+    @_advance_time ||= { last: nil, n: 0, avg: nil }
+    if n <= 1 || @_advance_time[:n] > n
+      @_advance_time[:last] = nil
+      @_advance_time[:n] = 0
+      @_advance_time[:avg]  = nil
+    end
+
+    # Estimate timing
+    unless total.nil? || @_advance_time[:last].nil? || @_advance_time[:n] == n
+      this_time = Time.now - @_advance_time[:last]
+      @_advance_time[:avg] ||= this_time / (n - @_advance_time[:n])
+      @_advance_time[:avg] = 0.99 * @_advance_time[:avg] + 0.01 * this_time
+    end
+    @_advance_time[:last] = Time.now
+    @_advance_time[:n] = n
+
+    # Report
+    adv_vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
+    adv =
+      total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
+        ('%.1f%% (%s/%s)' % adv_vals)
+    left =
+      if @_advance_time[:avg].nil?
+        ''
+      else
+        left_time = @_advance_time[:avg] * (total - n) / 60
+        left_time < 0.01 ? '         ' :
+          left_time < 1 ? ('%.0fs left' % (left_time * 60)) :
+          left_time > 60 ? ('%.1fh left' % (left_time / 60)) :
+          ('%.1fm left' % left_time)
+      end
+    $stderr.print("[%s] %s %s %s    \r" % [Time.now, step, adv, left])
   end
 
   ##

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 16, 1]
+  VERSION = [0.7, 16, 3]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 10, 19)
+  VERSION_DATE = Date.new(2020, 10, 27)
 
   ##
   # Reference of MiGA.

data/scripts/project_stats.bash

@@ -12,7 +12,7 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
-miga doctor -P "$PROJECT" -v
+miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.16.1
+  version: 0.7.16.3
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-19 00:00:00.000000000 Z
+date: 2020-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons