miga-base 0.7.16.0 → 0.7.16.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ee2f247a9f26ddbf2fcf0fc85bb0ed51d0d4980e2e40297449665a0befca102
-  data.tar.gz: 9d2ea26b71d226f0cbddebb214912a82b432886efb72e8bb6523b4b2f5aa588c
+  metadata.gz: fcd30efdaa8cf3a8ae87b5cc658363c2d37c0beb3c13491595beb983b1629b32
+  data.tar.gz: '0441764548f4199a8f3df253c151dc856aa414f5a91d02126bcb893ee2fe6835'
 SHA512:
-  metadata.gz: 673a1f46b1e34e14b58a434be79ec92c2ed31e7225004bd3fe08daebdf1362603353b5dd8473de8923583ed9819b6fe54578a3e18ea33dd51a54e2fad0bb0755
-  data.tar.gz: 23e44862a13fa293ea72d2260fc9676e2438b4c0c1725c64863a23c144d9b8a27dfefa62da6234fc0e86a8e59ada90498801a5609baaf07357293807801ccf60
+  metadata.gz: 18e829f1cb4141df9515565bd19af32e67b9920689a75dd4b5f2f5a79d0a5b7d86ea425f64e7702766e09a14c5b70af4d07ccb16df94aeef3ba4ade54e9506ab
+  data.tar.gz: 72ad22fd3386214825c463edc9232f2deeae24d06257a523f73592fd2d9496e452d192d10f37cf493b52a936578c3111a1a8801a78494688d51fc75b01e489f2

data/bin/miga

@@ -7,4 +7,5 @@ $LOAD_PATH.push File.expand_path('../../lib', __FILE__)
 require 'miga'
 require 'miga/cli'
 
-MiGA::Cli.new(ARGV).launch
+MiGA::Cli.new(ARGV).launch(true)
+

data/lib/miga/cli/action/doctor.rb

@@ -7,6 +7,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   include MiGA::Cli::Action::Doctor::Base
 
   def parse_cli
+    cli.defaults = { threads: 1 }
     cli.defaults = Hash[@@OPERATIONS.keys.map { |i| [i, true] }]
     cli.parse do |opt|
       operation_n = Hash[@@OPERATIONS.map { |k, v| [v[0], k] }]
@@ -24,6 +25,10 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
         @@OPERATIONS.each_key { |i| cli[i] = false }
         cli[op_k] = true
       end
+      opt.on(
+        '-t', '--threads INT', Integer,
+        "Concurrent threads to use. By default: #{cli[:threads]}"
+      ) { |v| cli[:threads] = v }
     end
   end
 
@@ -59,11 +64,19 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform status operation with MiGA::Cli +cli+
   def check_status(cli)
     cli.say 'Updating metadata status'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      d.recalculate_status
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        cli.load_project.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          d.recalculate_status if k % cli[:threads] == i
+        end
+      end
     end
+    Process.waitall
     cli.say
   end
 
@@ -71,36 +84,59 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform databases operation with MiGA::Cli +cli+
   def check_db(cli)
     cli.say 'Checking integrity of databases'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      each_database_file(d) do |db_file, metric, result|
-        check_sqlite3_database(db_file, metric) do
-          cli.say("  > Removing malformed database from #{d.name}:#{result}   ")
-          File.unlink(db_file)
-          r = d.result(result) or next
-          [r.path(:done), r.path].each { |f| File.unlink(f) if File.exist?(f) }
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        p.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
+          each_database_file(d) do |db_file, metric, result|
+            check_sqlite3_database(db_file, metric) do
+              cli.say(
+                "  > Removing malformed database from #{d.name}:#{result}   "
+              )
+              File.unlink(db_file)
+              r = d.result(result) or next
+              [r.path(:done), r.path].each do |f|
+                File.unlink(f) if File.exist?(f)
+              end
+            end
+          end
         end
       end
     end
+    Process.waitall
     cli.say
   end
 
   ##
   # Perform bidirectional operation with MiGA::Cli +cli+
   def check_bidir(cli)
-    cli.say 'Checking that reference distances are bidirectional'
+    cli.say 'Checking if reference distances are bidirectional'
     ref_ds = cli.load_project.each_dataset.select(&:ref?)
     ref_names = ref_ds.map(&:name)
-    n, k = ref_ds.size, 0
-    ref_ds.each do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      saved = saved_targets(d)
-      next if saved.nil?
+    n = ref_ds.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        ref_ds.each do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
 
-      to_save = ref_names - saved
-      to_save.each { |k| save_bidirectional(cli.load_project.dataset(k), d) }
+          saved = saved_targets(d)
+          next if saved.nil?
+
+          (ref_names - saved).each do |k|
+            save_bidirectional(cli.load_project.dataset(k), d)
+          end
+        end
+      end
     end
+    Process.waitall
     cli.say
   end
 

data/lib/miga/cli/action/doctor/base.rb

@@ -134,10 +134,19 @@ module MiGA::Cli::Action::Doctor::Base
 
       data[0], data[1] = data[1], data[0]
       SQLite3::Database.new(db_file_b) do |conn|
-        conn.execute(
-          "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-          "values(?, ?, ?, ?, ?, ?)", data
-        )
+        attempts = 0
+        begin
+          attempts += 1
+          conn.execute(
+            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+            "values(?, ?, ?, ?, ?, ?)", data
+          )
+        rescue SQLite3::BusyException => e
+          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
+
+          sleep(1)
+          retry
+        end
       end
     end
   end

data/lib/miga/cli/action/get.rb

@@ -70,7 +70,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob = get_sub_cli
     p = cli.load_project
     glob.each do |sub_cli|
-      rd = create_remote_dataset(sub_cli)
+      rd = create_remote_dataset(sub_cli, p)
       next if rd.nil?
 
       if sub_cli[:get_md]
@@ -115,7 +115,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob
   end
 
-  def create_remote_dataset(sub_cli)
+  def create_remote_dataset(sub_cli, p)
     sub_cli.ensure_par(dataset: '-D', ids: '-I')
     unless sub_cli[:api_key].nil?
       ENV["#{sub_cli[:universe].to_s.upcase}_API_KEY"] = sub_cli[:api_key]

data/lib/miga/cli/action/get_db.rb

@@ -165,7 +165,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
   def check_target
     return false if cli[:overwrite]
 
-    file = File.expand_path(cli[:database], cli[:local])
+    file = File.expand_path(cli[:database].to_s, cli[:local])
     if Dir.exist? file
       warn "The target directory already exists: #{file}"
       true

data/lib/miga/cli/action/run.rb

@@ -8,7 +8,7 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
   def parse_cli
     cli.defaults = { try_load: false, thr: 1, env: false }
     cli.parse do |opt|
-      cli.opt_object(opt, [:project, :dataset_opt, :result])
+      cli.opt_object(opt, [:project, :dataset_opt, :result_opt])
       opt.on(
         '-t', '--threads INT', Integer,
         "Threads to use in the local run (by default: #{cli[:thr]})"
@@ -43,6 +43,9 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
       cli[:dataset] = nil
     end
 
+    # Use virtual result if not explicitly passed
+    cli[:result] ||= cli[:dataset] ? :d : :p
+
     # Load project
     p = cli.load_project
 

data/lib/miga/cli/opt_helper.rb

@@ -56,10 +56,11 @@ module MiGA::Cli::OptHelper
   # - :project_type To allow (optionally) a type of project
   # - :project_type_req To require a type of project
   # - :result To require a type of project or dataset result
+  # - :result_opt To allow (optionally) a type of result
   # - :result_dataset To require a type of dataset result
   # - :result_project To require a type of project result
-  # The options :result, :result_dataset, and :result_project are mutually
-  # exclusive
+  # The options :result, :result_opt, :result_dataset, and :result_project
+  # are mutually exclusive
   def opt_object(opt, what = [:project, :dataset])
     what.each do |w|
       case w
@@ -82,10 +83,10 @@ module MiGA::Cli::OptHelper
           "#{req}Type of #{obj}. Recognized types include:",
           *klass.KNOWN_TYPES.map { |k, v| "~ #{k}: #{v[:description]}" }
         ) { |v| self[:type] = v.downcase.to_sym }
-      when :result
+      when :result, :result_opt
         opt.on(
           '-r', '--result STRING',
-          '(Mandatory) Name of the result',
+          "#{"(Mandatory) " if w == :result}Name of the result",
           'Recognized names for dataset-specific results include:',
           *MiGA::Dataset.RESULT_DIRS.keys.map { |n| " ~ #{n}" },
           'Recognized names for project-wide results include:',

data/lib/miga/common.rb

@@ -52,10 +52,44 @@ class MiGA::MiGA
   # 1,000 otherwise.
   # The report goes to $stderr iff --verborse
   def advance(step, n = 0, total = nil, bin = true)
-    adv = total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-      ('%.1f%% (%s/%s)' % [100.0 * n / total,
-                           num_suffix(n, bin), num_suffix(total, bin)])
-    $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
+    # Initialize advance timing
+    @_advance_time ||= { last: nil, n: 0, avg: nil }
+    if n <= 1 || @_advance_time[:n] > n
+      @_advance_time[:last] = nil
+      @_advance_time[:n] = 0
+      @_advance_time[:avg]  = nil
+    end
+
+    # Estimate timing
+    adv_n = n - @_advance_time[:n]
+    unless total.nil? || @_advance_time[:last].nil? || adv_n <= 0
+      if adv_n.to_f/n > 0.001
+        this_time = Time.now - @_advance_time[:last]
+        this_avg = this_time / adv_n
+        @_advance_time[:avg] ||= this_avg
+        @_advance_time[:avg] = 0.9 * @_advance_time[:avg] + 0.1 * this_avg
+      end
+    end
+    @_advance_time[:last] = Time.now
+    @_advance_time[:n] = n
+
+    # Report
+    adv_vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
+    adv =
+      total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
+        ('%.1f%% (%s/%s)' % adv_vals)
+    left =
+      if @_advance_time[:avg].nil?
+        ''
+      else
+        left_time = @_advance_time[:avg] * (total - n) / 60 # <- in minutes
+        left_time < 0.01 ? '         ' :
+          left_time < 1 ? ('%.0fs left' % (left_time * 60)) :
+          left_time > 1440 ? ('%.1fd left' % (left_time / 1440)) :
+          left_time > 60 ? ('%.1fh left' % (left_time / 60)) :
+          ('%.1fm left' % left_time)
+      end
+    $stderr.print("[%s] %s %s %s    \r" % [Time.now, step, adv, left])
   end
 
   ##

data/lib/miga/daemon/base.rb

@@ -77,6 +77,7 @@ module MiGA::Daemon::Base
   ##
   # Writing file handler (IO) to the log file
   def logfh
+    @logfh ||= nil
     return $stderr if show_log?
     return @logfh if @logfh && !@logfh.closed?
 

data/lib/miga/json.rb

@@ -34,17 +34,29 @@ class MiGA::Json < MiGA::MiGA
     # +opts+.
     def parse(path, opts = {})
       opts = default_opts(opts)
-      cont = opts[:contents] ? path : File.read(path)
-      raise "Empty descriptor: #{opts[:contents] ? "''" : path}." if cont.empty?
 
-      y = JSON.parse(cont,
-                     symbolize_names: opts[:symbolize],
-                     create_additions: opts[:additions])
+      # Read JSON
+      cont = path
+      12.times do
+        cont = File.read(path)
+        break unless cont.empty?
+        sleep 1 # Wait up to 12 seconds for racing processes (iff empty file)
+      end unless opts[:contents]
+      raise "Empty descriptor: #{opts[:contents] ? "''" : path}" if cont.empty?
+
+      # Parse JSON
+      params = { symbolize_names: opts[:symbolize],
+                 create_additions: opts[:additions] }
+      y = JSON.parse(cont, params)
+
+      # Add defaults
       unless opts[:default].nil?
         opts[:default] = parse(opts[:default]) if opts[:default].is_a? String
         y.each { |k, v| opts[:default][k] = v }
         y = opts[:default]
       end
+
+      # Return
       y
     end
 

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 16, 0]
+  VERSION = [0.7, 16, 6]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 10, 13)
+  VERSION_DATE = Date.new(2020, 11, 22)
 
   ##
   # Reference of MiGA.

data/scripts/project_stats.bash

@@ -12,7 +12,7 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
-miga doctor -P "$PROJECT" -v
+miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.16.0
+  version: 0.7.16.6
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-13 00:00:00.000000000 Z
+date: 2020-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons