miga-base 0.7.16.10 → 0.7.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80bc5ff58583017ef3cb4962c7a2ac4086e602fc9bd2319a35cf7edd36c211d3
-  data.tar.gz: 6ae15844eff00f4adcf794f7b56f940cc4af96a555785a1397dfd3781e62af80
+  metadata.gz: bc5df224531c2f5e902304b478aabe0ba18c02b836c5bab66a675257a3739a08
+  data.tar.gz: 438b9f2cefad3ecc3b65e17f81d5708f6a589c8b34f22b63235cf4123a156922
 SHA512:
-  metadata.gz: bfb11a9ffe5a5b8d64578512cf5ed29a6272f90760015092a35d2e17a008cd84bdf0c2c30f6988941815b196e4c492abf7b5929166b2ef6357297e3784a91936
-  data.tar.gz: 77846f20c76d8ba84b3c1351395da6cc410e8947bb82491d9e93c7f1a1931c22ec975eb5ffc94117524cddf83c30890ea84bbabe7585783aa2034274093c7e88
+  metadata.gz: 506dd5c19259071abd5cf0a130a70557967833edc520fada73030372d7736f2fe114ce4879bc224e37bcbe407a566bbd3cef39a5bc8ec9669c9c4cd1e63edb74
+  data.tar.gz: 4edb685f9a4adb69339e5ef477d4a0a332da606b402d380e3eab77a055ab94582be3be90f904e9403bc20d3f81c3753ed85df8a43bd9afc3e97b41f88cfd9fb1

data/lib/miga/cli/action/doctor/base.rb

@@ -1,5 +1,5 @@
 require 'miga/cli/action'
-require 'sqlite3'
+require 'miga/sqlite'
 
 class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
 end
@@ -10,9 +10,7 @@ module MiGA::Cli::Action::Doctor::Base
   # tables saving +metric+ (:ani or :aai) and call +blk+ if the
   # file is corrupt or doesn't contain the expected structure
   def check_sqlite3_database(db_file, metric, &blk)
-    SQLite3::Database.new(db_file) do |conn|
-      conn.execute("select count(*) from #{metric}").first
-    end
+    MiGA::SQLite.new(db_file).run("select count(*) from #{metric}")
   rescue SQLite3::SQLException, SQLite3::CorruptException
     blk.call
   end
@@ -107,11 +105,7 @@ module MiGA::Cli::Action::Doctor::Base
     dist = dataset.result(:distances) or return
     path = dist.file_path(:aai_db) or return
 
-    o = []
-    SQLite3::Database.new(path) do |conn|
-      o = conn.execute('select seq2 from aai').map(&:first)
-    end
-    o
+    MiGA::SQLite.new(path).run('select seq2 from aai').map(&:first)
   end
 
   ##
@@ -120,34 +114,20 @@ module MiGA::Cli::Action::Doctor::Base
   def save_bidirectional(a, b)
     each_database_file(a) do |db_file, metric, result|
       data = nil
-      SQLite3::Database.new(db_file) do |conn|
-        data =
-          conn.execute(
-            "select seq1, seq2, #{metric}, sd, n, omega " +
-            "from #{metric} where seq2 = ? limit 1", b.name
-          ).first
-      end
+      data = MiGA::SQLite.new(db_file).run(
+        "select seq1, seq2, #{metric}, sd, n, omega " +
+        "from #{metric} where seq2 = ? limit 1", b.name
+      ).first
       next if data.nil? || data.empty?
 
       db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
       next unless File.exist?(db_file_b)
 
       data[0], data[1] = data[1], data[0]
-      SQLite3::Database.new(db_file_b) do |conn|
-        attempts = 0
-        begin
-          attempts += 1
-          conn.execute(
-            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-            "values(?, ?, ?, ?, ?, ?)", data
-          )
-        rescue SQLite3::BusyException => e
-          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
-
-          sleep(1)
-          retry
-        end
-      end
+      MiGA::SQLite.new(db_file_b).run(
+        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+        "values(?, ?, ?, ?, ?, ?)", data
+      )
     end
   end
 end

data/lib/miga/cli/action/lair.rb

@@ -25,6 +25,10 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
         '-p', '--path PATH',
         '(Mandatory) Path to the directory where the MiGA projects are located'
       ) { |v| cli[:path] = v }
+      opt.on(
+        '--exclude NAME1,NAME2,NAME3', Array,
+        'Exclude these projects (identified by name) from the lair'
+      ) { |v| cli[:exclude] = v }
       opt.on(
         '--json PATH',
         'Path to a custom daemon definition in json format'
@@ -80,19 +84,22 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
 
   def perform
     cli.ensure_par(path: '-p')
+    k_opts = %i[
+      json latency wait_for keep_inactive trust_timestamp name dry exclude
+    ]
+    opts = Hash[k_opts.map { |k| [k, cli[k]] }]
+    lair = MiGA::Lair.new(cli[:path], opts)
+
     case cli.operation.to_sym
     when :terminate
-      MiGA::Lair.new(cli[:path]).terminate_daemons
+      lair.terminate_daemons
     when :list
       o = []
-      MiGA::Lair.new(cli[:path]).each_daemon do |d|
+      lair.each_daemon do |d|
         o << [d.daemon_name, d.class, d.daemon_home, d.active?, d.last_alive]
       end
       cli.table(%w[name class path active last_alive], o)
     else
-      k_opts = %i[json latency wait_for keep_inactive trust_timestamp name dry]
-      opts = Hash[k_opts.map { |k| [k, cli[k]] }]
-      lair = MiGA::Lair.new(cli[:path], opts)
       lair.daemon(cli.operation, cli[:daemon_opts])
     end
   end

data/lib/miga/cli/action/run.rb

@@ -59,7 +59,7 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
            "MIGA=#{miga.shellescape}", "CORES=#{cli[:thr]}"]
     obj = cli.load_project_or_dataset
     klass = obj.class
-    virtual_task = [:p, :d].include?(cli[:result])
+    virtual_task = %i[p d maintenance].include?(cli[:result])
     cmd << "DATASET=#{obj.name.shellescape}" if obj.is_a? MiGA::Dataset
     if klass.RESULT_DIRS[cli[:result]].nil? and not virtual_task
       raise "Unsupported #{klass.to_s.sub(/.*::/, '')} result: #{cli[:result]}."

data/lib/miga/cli/action/wf.rb

@@ -184,7 +184,7 @@ module MiGA::Cli::Action::Wf
   def call_cli(cmd)
     cmd << '-v' if cli[:verbose]
     MiGA::MiGA.DEBUG "Cli::Action::Wf.call_cli #{cmd}"
-    MiGA::Cli.new(cmd.map(&:to_s)).launch
+    MiGA::Cli.new(cmd.map(&:to_s)).launch(true)
   end
 
   def run_daemon

data/lib/miga/common.rb

@@ -74,10 +74,13 @@ class MiGA::MiGA
     @_advance_time[:n] = n
 
     # Report
-    adv_vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
     adv =
-      total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-        ('%.1f%% (%s/%s)' % adv_vals)
+      if total.nil?
+        (n == 0 ? '' : num_suffix(n, bin))
+      else
+        vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
+        ('%.1f%% (%s/%s)' % vals)
+      end
     left =
       if @_advance_time[:avg].nil?
         ''

data/lib/miga/common/with_daemon.rb

@@ -158,7 +158,7 @@ module MiGA::Common::WithDaemon
   # Returns the status of the daemon with +opts+
   def status(opts = [], wait = true)
     if active?
-      say "Running with pid #{File.read(pid_file)}"
+      say "Running with pid #{File.size?(pid_file) ? File.read(pid_file) : '?'}"
     else
       say 'Not running'
     end

data/lib/miga/daemon.rb

@@ -73,7 +73,7 @@ class MiGA::Daemon < MiGA::MiGA
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
     miga_say "Saving log to: #{output_file}" unless show_log?
-    recalculate_status!
+    queue_maintenance
     load_status
     say 'Configuration options:'
     say @runopts.to_s
@@ -92,8 +92,7 @@ class MiGA::Daemon < MiGA::MiGA
     flush!
     if (loop_i % 12).zero?
       purge!
-      # TEMPORARILY DISABLED:
-      # recalculate_status!
+      queue_maintenance
     end
     save_status
     sleep(latency)
@@ -101,9 +100,13 @@ class MiGA::Daemon < MiGA::MiGA
     true
   end
 
-  def recalculate_status!
-    say 'Recalculating status for all datasets'
-    project.each_dataset(&:recalculate_status)
+  ##
+  # Queue maintenance tasks as an analysis job
+  def queue_maintenance
+    return if bypass_maintenance?
+
+    say 'Queueing maintenance tasks'
+    queue_job(:maintenance)
   end
 
   ##
@@ -179,6 +182,7 @@ class MiGA::Daemon < MiGA::MiGA
     unless show_log?
       n = project.dataset_names.count
       k = jobs_to_run.size + jobs_running.size
+      k -= 1 unless get_job(:maintenance).nil?
       advance('Datasets:', n - k, n, false)
       miga_say if k == 0
     end
@@ -271,6 +275,11 @@ class MiGA::Daemon < MiGA::MiGA
     # Avoid single datasets hogging resources
     @jobs_to_run.rotate! rand(jobs_to_run.size)
 
+    # Prioritize: Project-wide > MiGA Online queries > Other datasets
+    @jobs_to_run.sort_by! do |job|
+      job[:ds].nil? ? 1 : job[:ds_name] =~ /^qG_/ ? 2 : 3
+    end
+
     # Launch as many +jobs_to_run+ as possible
     while (hostk = next_host)
       break if jobs_to_run.empty?
@@ -309,11 +318,13 @@ class MiGA::Daemon < MiGA::MiGA
       job[:hostk] = hostk
       job[:cmd] = job[:cmd].miga_variables(host: nodelist[hostk])
       job[:pid] = spawn job[:cmd]
-      Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
+      MiGA::MiGA.DEBUG "Detaching PID: #{job[:pid]}"
+      Process.detach(job[:pid]) unless [nil, '', 0].include?(job[:pid])
     when 'bash'
       # Local job
       job[:pid] = spawn job[:cmd]
-      Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
+      MiGA::MiGA.DEBUG "Detaching PID: #{job[:pid]}"
+      Process.detach(job[:pid]) unless [nil, '', 0].include?(job[:pid])
     else
       # Schedule cluster job (qsub, msub, slurm)
       job[:pid] = `#{job[:cmd]}`.chomp

data/lib/miga/daemon/base.rb

@@ -17,7 +17,7 @@ module MiGA::Daemon::Base
         if !force && v == 0 && k != :verbosity
           raise "Daemon's #{k} cannot be set to zero"
         end
-      when :shutdown_when_done, :show_log
+      when :shutdown_when_done, :show_log, :bypass_maintenance
         v = !!v
       when :nodelist
         if v =~ /^\$/
@@ -63,6 +63,12 @@ module MiGA::Daemon::Base
     !!runopts(:shutdown_when_done)
   end
 
+  ##
+  # Should the daemon ignore regular maintenance steps?
+  def bypass_maintenance?
+    !!runopts(:bypass_maintenance)
+  end
+
   ##
   # Returns the level of verbosity for the daemon as an Integer, or 1 if unset.
   # Verbosity levels are:

data/lib/miga/dataset.rb

@@ -7,7 +7,11 @@ require 'miga/metadata'
 require 'miga/dataset/result'
 require 'miga/dataset/status'
 require 'miga/dataset/hooks'
-require 'sqlite3'
+
+# This library is only required by +#closest_relatives+, so it is now
+# being loaded on call instead to allow most of miga-base to work without
+# issue in systems with problematic SQLite3 installations.
+# require 'miga/sqlite'
 
 ##
 # Dataset representation in MiGA
@@ -190,8 +194,8 @@ class MiGA::Dataset < MiGA::MiGA
     r = result(ref_project ? :taxonomy : :distances)
     return nil if r.nil?
 
-    db = SQLite3::Database.new(r.file_path(:aai_db))
-    db.execute(
+    require 'miga/sqlite'
+    MiGA::SQLite.new(r.file_path(:aai_db)).run(
       'SELECT seq2, aai FROM aai WHERE seq2 != ? ' \
       'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many]
     )

data/lib/miga/dataset/result.rb

@@ -1,8 +1,12 @@
-require 'sqlite3'
 require 'miga/result'
 require 'miga/dataset/base'
 require 'miga/common/with_result'
 
+# This library is only required by +#cleanup_distances!+, so it is now
+# being loaded on call instead to allow most of miga-base to work without
+# issue in systems with problematic SQLite3 installations.
+# require 'miga/sqlite'
+
 ##
 # Helper module including specific functions to add dataset results
 module MiGA::Dataset::Result
@@ -144,20 +148,21 @@ module MiGA::Dataset::Result
   # the project as reference datasets.
   def cleanup_distances!
     r = get_result(:distances)
-    ref = project.datasets.select(&:ref?).select(&:active?).map(&:name)
     return if r.nil?
 
+    require 'miga/sqlite'
+    ref = project.datasets.select(&:ref?).select(&:active?).map(&:name)
     %i[haai_db aai_db ani_db].each do |db_type|
       db = r.file_path(db_type)
       next if db.nil? || !File.size?(db)
 
-      sqlite_db = SQLite3::Database.new db
+      sqlite_db = MiGA::SQLite.new(db)
       table = db_type[-6..-4]
-      val = sqlite_db.execute "select seq2 from #{table}"
+      val = sqlite_db.run("select seq2 from #{table}")
       next if val.empty?
 
       (val.map(&:first) - ref).each do |extra|
-        sqlite_db.execute "delete from #{table} where seq2=?", extra
+        sqlite_db.run("delete from #{table} where seq2=?", extra)
       end
     end
   end

data/lib/miga/lair.rb

@@ -31,6 +31,7 @@ class MiGA::Lair < MiGA::MiGA
   #   by default: true
   # - dry: Only report when daemons would be launched, but don't actually launch
   #   them
+  # - exclude: Array of project names to be excluded from the lair
   def initialize(path, opts = {})
     @path = File.expand_path(path)
     @options = opts
@@ -41,7 +42,8 @@ class MiGA::Lair < MiGA::MiGA
       keep_inactive: false,
       trust_timestamp: true,
       name: File.basename(@path),
-      dry: false
+      dry: false,
+      exclude: []
     }.each { |k, v| @options[k] = v if @options[k].nil? }
   end
 
@@ -109,7 +111,7 @@ class MiGA::Lair < MiGA::MiGA
         project = MiGA::Project.load(f)
         raise "Cannot load project: #{f}" if project.nil?
 
-        yield(project)
+        yield(project) unless options[:exclude].include?(project.name)
       elsif Dir.exist? f
         each_project(f) { |p| yield(p) }
       end

data/lib/miga/sqlite.rb

@@ -0,0 +1,49 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'sqlite3'
+
+##
+# SQLite3 wrapper for MiGA.
+class MiGA::SQLite < MiGA::MiGA
+  class << self
+    ##
+    # Default parsing options. Supported +opts+ keys:
+    # - +:busy_attempts+: Number of times to retry when database is busy
+    #   (default: 3)
+    def default_opts(opts = {})
+      opts[:busy_attempts] ||= 3
+      opts
+    end
+  end
+
+  ##
+  # Options hash
+  attr :opts
+
+  ##
+  # Database absolute path
+  attr :path
+
+  ##
+  # Create MiGA::SQLite with database in +path+ (without opening a connection)
+  # and options +opts+ (see +.default_opts+)
+  def initialize(path, opts = {})
+    @opts = MiGA::SQLite.default_opts(opts)
+    @path = File.absolute_path(path)
+  end
+
+  ##
+  # Executes +cmd+ and returns the result
+  def run(*cmd)
+    busy_attempts ||= 0
+    conn = SQLite3::Database.new(path)
+    conn.execute(*cmd)
+  rescue SQLite3::BusyException => e
+    busy_attempts += 1
+    raise "Database busy #{path}: #{e.message}" if busy_attempts >= 3
+
+    sleep(1)
+    retry
+  end
+end

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 16, 10]
+  VERSION = [0.7, 19, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 12, 9)
+  VERSION_DATE = Date.new(2021, 1, 2)
 
   ##
   # Reference of MiGA.

data/scripts/maintenance.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
+SCRIPT="maintenance"
+# shellcheck source=scripts/miga.bash
+. "$MIGA/scripts/miga.bash" || exit 1
+
+miga doctor --only status -P "$PROJECT" -t "$CORES" -v
+

data/scripts/miga.bash

@@ -11,17 +11,17 @@ function exists { [[ -e "$1" ]] ; }
 function fx_exists { [[ $(type -t "$1") == "function" ]] ; }
 
 if [[ "$SCRIPT" != "d" && "$SCRIPT" != "p" ]] ; then
-  echo '############'
-  echo -n "Date: " ; miga date
-  echo "Hostname: $(hostname)"
-  echo "MiGA: $MIGA"
-  echo "Task: $SCRIPT"
-  echo "Project: $PROJECT"
+  echo ""
+  echo "######[ $SCRIPT ]######"
+  echo "# Date: $(miga date)"
+  echo "# Host: $(hostname)"
+  echo "# MiGA: $MIGA"
+  echo "# Project: $PROJECT"
   if [[ -n $DATASET ]] ; then
-    echo "Dataset: $DATASET"
+    echo "# Dataset: $DATASET"
     miga edit -P "$PROJECT" -D "$DATASET" -m "_step=$SCRIPT"
   fi
-  echo '------------'
+  echo '#------------'
 fi
 
 true

data/scripts/project_stats.bash

@@ -12,12 +12,15 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
+echo "# Doctor"
 miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
+echo "# Index taxonomy"
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active
 
 # Index metadata
+echo "# Index metadata"
 ruby -I "$MIGA/lib" \
   "$MIGA/utils/index_metadata.rb" "$PROJECT" "miga-project.metadata.db"
 

data/test/daemon_test.rb

@@ -93,7 +93,7 @@ class DaemonTest < Test::Unit::TestCase
       0 => /-{20}\n/,
       1 => /MiGA:#{p.name} launched/,
       2 => /-{20}\n/,
-      6 => /Probing running jobs\n/
+      8 => /Probing running jobs\n/
     }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
   ensure
     begin
@@ -160,8 +160,9 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d1.jobs_running.size)
     assert_equal(0, d1.jobs_to_run.size)
     capture_stderr { d1.in_loop }
+    # 3 dataset jobs + 1 maintenance job:
     assert_equal(1, d1.jobs_running.size)
-    assert_equal(2, d1.jobs_to_run.size)
+    assert_equal(3, d1.jobs_to_run.size)
   end
 
   def test_maxjobs_runopts
@@ -172,7 +173,7 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d1.jobs_to_run.size)
     capture_stderr { d1.in_loop }
     assert_equal(2, d1.jobs_running.size)
-    assert_equal(1, d1.jobs_to_run.size)
+    assert_equal(2, d1.jobs_to_run.size)
   end
 
   def test_load_status
@@ -232,6 +233,7 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_shutdown_when_done
+    daemon.runopts(:bypass_maintenance, true)
     daemon.runopts(:shutdown_when_done, true)
     out = capture_stderr { assert { !daemon.in_loop } }.string
     assert_match(/Nothing else to do/, out)
@@ -317,4 +319,20 @@ class DaemonTest < Test::Unit::TestCase
     out = capture_stderr { d1.in_loop }.string
     assert_match(/Daemon loop start/, out)
   end
+
+  def test_bypass_maintenance
+    # Default (run maintenance)
+    d = daemon(0)
+    d.runopts(:latency, 0, true)
+    capture_stderr { d.in_loop }
+    assert_equal(1, d.jobs_running.size)
+    assert_equal(:maintenance, d.jobs_running.first[:job])
+
+    # Bypassing maintenance
+    d = daemon(1)
+    d.runopts(:latency, 0, true)
+    d.runopts(:bypass_maintenance, true)
+    capture_stderr { d.in_loop }
+    assert_equal([], d.jobs_running)
+  end
 end

data/utils/cleanup-databases.rb

@@ -5,21 +5,17 @@ require 'miga'
 
 ARGV[1] or abort "Usage: #{$0} path/to/project threads"
 
-$stderr.puts 'Cleaning databases...'
 p = MiGA::Project.load(ARGV[0])
-ds_names = p.dataset_names
+dsn = p.dataset_names
 thr = ARGV[1].to_i
 
-pc = [0] + (1..100).map { |i| ds_names.size * i / 100 }
-$stderr.puts (('.' * 9 + '|') * 10) + ' 100%'
+m = MiGA::MiGA.new
+m.say 'Cleaning Databases'
 
 (0..thr - 1).each do |t|
   fork do
-    ds_names.each_with_index do |i, idx|
-      while t == 0 and idx + 1 > pc.first
-        $stderr.print '#'
-        pc.shift
-      end
+    dsn.each_with_index do |i, idx|
+      m.advance('Dataset:', dsn.size, idx + 1) if t == 0
       next unless (idx % thr) == t
 
       d = p.dataset(i)
@@ -30,4 +26,6 @@ $stderr.puts (('.' * 9 + '|') * 10) + ' 100%'
   end
 end
 Process.waitall
-$stderr.puts ' Done'
+m.advance('Dataset:', dsn.size, dsn.size)
+m.say
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.16.10
+  version: 0.7.19.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-09 00:00:00.000000000 Z
+date: 2021-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -196,6 +196,7 @@ files:
 - lib/miga/result/dates.rb
 - lib/miga/result/source.rb
 - lib/miga/result/stats.rb
+- lib/miga/sqlite.rb
 - lib/miga/tax_dist.rb
 - lib/miga/tax_index.rb
 - lib/miga/taxonomy.rb
@@ -211,6 +212,7 @@ files:
 - scripts/essential_genes.bash
 - scripts/haai_distances.bash
 - scripts/init.bash
+- scripts/maintenance.bash
 - scripts/miga.bash
 - scripts/mytaxa.bash
 - scripts/mytaxa_scan.bash