miga-base 0.7.16.1 → 0.7.16.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c515f327c9d7178b1891283f849d422885b3640f8cded39cfccd048e354acf16
-  data.tar.gz: 889a399e0fa68b54c901ec513e2049ab046593bc433185511516dbf9b3cedb7d
+  metadata.gz: affb99901724300ab46e2e7b1ef3f4e227f4288bef3a0634c8caa5f86b10b345
+  data.tar.gz: fc51e29cc954afd22ae6ab59c59393165a3ea64455807bac8e2300660f2c2673
 SHA512:
-  metadata.gz: f4a9d5515ddf4b208b81612ceafcdae0eeb9cd37266df5f0175cef859adb9b24fd6e9ac5adf384ad9c91f8ecc08d6d5522eef8b9dd96e7d39740b14f83fdaf53
-  data.tar.gz: 117bc5b917e1008084e064cd4d87f290a83b6962e93560abfad254d23345a7685bf9ad3a345f2ade98ea5c13b4be52ccf608683ede8c320f3b8ee1250c2cec32
+  metadata.gz: f8c9f1c7c307e408080ea80a4496eba48fe9d05220b6e636490958358c5a19da1f2ee86971c4b02b8200fcd8d5f01ec949b62dfbfce7b70b68da8ce940c7f5bf
+  data.tar.gz: cc4fed1bdef75754f4110dc768cc87b4766e6f7cf91166fdfa321657952066831b39b6da10aa598929a0155c93f555ba7ed0c0afe3dc118a92a7c954ce271551

data/bin/miga

@@ -7,4 +7,5 @@ $LOAD_PATH.push File.expand_path('../../lib', __FILE__)
 require 'miga'
 require 'miga/cli'
 
-MiGA::Cli.new(ARGV).launch
+MiGA::Cli.new(ARGV).launch(true)
+

data/lib/miga/cli/action/doctor.rb

@@ -7,6 +7,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   include MiGA::Cli::Action::Doctor::Base
 
   def parse_cli
+    cli.defaults = { threads: 1 }
     cli.defaults = Hash[@@OPERATIONS.keys.map { |i| [i, true] }]
     cli.parse do |opt|
       operation_n = Hash[@@OPERATIONS.map { |k, v| [v[0], k] }]
@@ -24,6 +25,10 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
         @@OPERATIONS.each_key { |i| cli[i] = false }
         cli[op_k] = true
       end
+      opt.on(
+        '-t', '--threads INT', Integer,
+        "Concurrent threads to use. By default: #{cli[:threads]}"
+      ) { |v| cli[:threads] = v }
     end
   end
 
@@ -59,11 +64,19 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform status operation with MiGA::Cli +cli+
   def check_status(cli)
     cli.say 'Updating metadata status'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      d.recalculate_status
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        cli.load_project.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          d.recalculate_status if k % cli[:threads] == i
+        end
+      end
     end
+    Process.waitall
     cli.say
   end
 
@@ -71,36 +84,59 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform databases operation with MiGA::Cli +cli+
   def check_db(cli)
     cli.say 'Checking integrity of databases'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      each_database_file(d) do |db_file, metric, result|
-        check_sqlite3_database(db_file, metric) do
-          cli.say("  > Removing malformed database from #{d.name}:#{result}   ")
-          File.unlink(db_file)
-          r = d.result(result) or next
-          [r.path(:done), r.path].each { |f| File.unlink(f) if File.exist?(f) }
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        p.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
+          each_database_file(d) do |db_file, metric, result|
+            check_sqlite3_database(db_file, metric) do
+              cli.say(
+                "  > Removing malformed database from #{d.name}:#{result}   "
+              )
+              File.unlink(db_file)
+              r = d.result(result) or next
+              [r.path(:done), r.path].each do |f|
+                File.unlink(f) if File.exist?(f)
+              end
+            end
+          end
         end
       end
     end
+    Process.waitall
     cli.say
   end
 
   ##
   # Perform bidirectional operation with MiGA::Cli +cli+
   def check_bidir(cli)
-    cli.say 'Checking that reference distances are bidirectional'
+    cli.say 'Checking if reference distances are bidirectional'
     ref_ds = cli.load_project.each_dataset.select(&:ref?)
     ref_names = ref_ds.map(&:name)
-    n, k = ref_ds.size, 0
-    ref_ds.each do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      saved = saved_targets(d)
-      next if saved.nil?
+    n = ref_ds.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        ref_ds.each do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
 
-      to_save = ref_names - saved
-      to_save.each { |k| save_bidirectional(cli.load_project.dataset(k), d) }
+          saved = saved_targets(d)
+          next if saved.nil?
+
+          (ref_names - saved).each do |k|
+            save_bidirectional(cli.load_project.dataset(k), d)
+          end
+        end
+      end
     end
+    Process.waitall
     cli.say
   end
 

data/lib/miga/cli/action/doctor/base.rb

@@ -134,10 +134,19 @@ module MiGA::Cli::Action::Doctor::Base
 
       data[0], data[1] = data[1], data[0]
       SQLite3::Database.new(db_file_b) do |conn|
-        conn.execute(
-          "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-          "values(?, ?, ?, ?, ?, ?)", data
-        )
+        attempts = 0
+        begin
+          attempts += 1
+          conn.execute(
+            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+            "values(?, ?, ?, ?, ?, ?)", data
+          )
+        rescue SQLite3::BusyException => e
+          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
+
+          sleep(1)
+          retry
+        end
       end
     end
   end

data/lib/miga/cli/action/get.rb

@@ -70,7 +70,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob = get_sub_cli
     p = cli.load_project
     glob.each do |sub_cli|
-      rd = create_remote_dataset(sub_cli)
+      rd = create_remote_dataset(sub_cli, p)
       next if rd.nil?
 
       if sub_cli[:get_md]
@@ -115,7 +115,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob
   end
 
-  def create_remote_dataset(sub_cli)
+  def create_remote_dataset(sub_cli, p)
     sub_cli.ensure_par(dataset: '-D', ids: '-I')
     unless sub_cli[:api_key].nil?
       ENV["#{sub_cli[:universe].to_s.upcase}_API_KEY"] = sub_cli[:api_key]

data/lib/miga/cli/action/get_db.rb

@@ -165,7 +165,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
   def check_target
     return false if cli[:overwrite]
 
-    file = File.expand_path(cli[:database], cli[:local])
+    file = File.expand_path(cli[:database].to_s, cli[:local])
     if Dir.exist? file
       warn "The target directory already exists: #{file}"
       true

data/lib/miga/cli/action/run.rb

@@ -8,7 +8,7 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
   def parse_cli
     cli.defaults = { try_load: false, thr: 1, env: false }
     cli.parse do |opt|
-      cli.opt_object(opt, [:project, :dataset_opt, :result])
+      cli.opt_object(opt, [:project, :dataset_opt, :result_opt])
       opt.on(
         '-t', '--threads INT', Integer,
         "Threads to use in the local run (by default: #{cli[:thr]})"
@@ -37,12 +37,18 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
       cli[:thr] ||= ENV['CORES'].to_i unless ENV['CORES'].nil?
       cli[:result] = File.basename(cli[:result].to_s, '.bash').to_sym
     end
+    cli[:project] = nil if cli[:project].empty?
+    cli[:dataset] = nil if cli[:dataset].empty?
+    cli[:result] = nil if cli[:result].empty?
 
     # Unset dataset if the requested result is for projects
     if (MiGA::Project.RESULT_DIRS.keys + [:p]).include? cli[:result]
       cli[:dataset] = nil
     end
 
+    # Use virtual result if not explicitly passed
+    cli[:result] ||= cli[:dataset] ? :d : :p
+
     # Load project
     p = cli.load_project
 

data/lib/miga/cli/opt_helper.rb

@@ -56,10 +56,11 @@ module MiGA::Cli::OptHelper
   # - :project_type To allow (optionally) a type of project
   # - :project_type_req To require a type of project
   # - :result To require a type of project or dataset result
+  # - :result_opt To allow (optionally) a type of result
   # - :result_dataset To require a type of dataset result
   # - :result_project To require a type of project result
-  # The options :result, :result_dataset, and :result_project are mutually
-  # exclusive
+  # The options :result, :result_opt, :result_dataset, and :result_project
+  # are mutually exclusive
   def opt_object(opt, what = [:project, :dataset])
     what.each do |w|
       case w
@@ -82,10 +83,10 @@ module MiGA::Cli::OptHelper
           "#{req}Type of #{obj}. Recognized types include:",
           *klass.KNOWN_TYPES.map { |k, v| "~ #{k}: #{v[:description]}" }
         ) { |v| self[:type] = v.downcase.to_sym }
-      when :result
+      when :result, :result_opt
         opt.on(
           '-r', '--result STRING',
-          '(Mandatory) Name of the result',
+          "#{"(Mandatory) " if w == :result}Name of the result",
           'Recognized names for dataset-specific results include:',
           *MiGA::Dataset.RESULT_DIRS.keys.map { |n| " ~ #{n}" },
           'Recognized names for project-wide results include:',

data/lib/miga/common.rb

@@ -52,10 +52,44 @@ class MiGA::MiGA
   # 1,000 otherwise.
   # The report goes to $stderr iff --verborse
   def advance(step, n = 0, total = nil, bin = true)
-    adv = total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-      ('%.1f%% (%s/%s)' % [100.0 * n / total,
-                           num_suffix(n, bin), num_suffix(total, bin)])
-    $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
+    # Initialize advance timing
+    @_advance_time ||= { last: nil, n: 0, avg: nil }
+    if n <= 1 || @_advance_time[:n] > n
+      @_advance_time[:last] = nil
+      @_advance_time[:n] = 0
+      @_advance_time[:avg]  = nil
+    end
+
+    # Estimate timing
+    adv_n = n - @_advance_time[:n]
+    unless total.nil? || @_advance_time[:last].nil? || adv_n <= 0
+      if adv_n.to_f/n > 0.001
+        this_time = Time.now - @_advance_time[:last]
+        this_avg = this_time / adv_n
+        @_advance_time[:avg] ||= this_avg
+        @_advance_time[:avg] = 0.9 * @_advance_time[:avg] + 0.1 * this_avg
+      end
+    end
+    @_advance_time[:last] = Time.now
+    @_advance_time[:n] = n
+
+    # Report
+    adv_vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
+    adv =
+      total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
+        ('%.1f%% (%s/%s)' % adv_vals)
+    left =
+      if @_advance_time[:avg].nil?
+        ''
+      else
+        left_time = @_advance_time[:avg] * (total - n) / 60 # <- in minutes
+        left_time < 0.01 ? '         ' :
+          left_time < 1 ? ('%.0fs left' % (left_time * 60)) :
+          left_time > 1440 ? ('%.1fd left' % (left_time / 1440)) :
+          left_time > 60 ? ('%.1fh left' % (left_time / 60)) :
+          ('%.1fm left' % left_time)
+      end
+    $stderr.print("[%s] %s %s %s    \r" % [Time.now, step, adv, left])
   end
 
   ##

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 16, 1]
+  VERSION = [0.7, 16, 7]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 10, 19)
+  VERSION_DATE = Date.new(2020, 11, 22)
 
   ##
   # Reference of MiGA.

data/scripts/project_stats.bash

@@ -12,7 +12,7 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
-miga doctor -P "$PROJECT" -v
+miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.16.1
+  version: 0.7.16.7
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-19 00:00:00.000000000 Z
+date: 2020-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons