miga-base 0.7.16.8 → 0.7.18.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ede9e4d8b2ae673b55e0b28f9e78292c42791c4ad400a02ef2c9f60065c1fc26
-  data.tar.gz: ff42e5a4f220774b7863f528fd334fa73a1acca0afcbb1a7f5e32dedd15bd1c7
+  metadata.gz: 1a0c77da5b4e0cfaee8d470f669d0589f00949c686868780c9c8ee2eb68cd9cd
+  data.tar.gz: 9f78d665d2559eacc0a19dd16d8b3cd712a512909db8c1a7af1dcb0edd7302d4
 SHA512:
-  metadata.gz: d02bc826d5dee9c84705d2f0fb30e1902cb5014385efe350fa2593347cd74f10007475cd3b5610a92fa9a93cf00378c508af7bcd36fb6ed3e60faad96b35eb49
-  data.tar.gz: c094b4bb67eddc15345cf53866b03154a85b39de09c10d8ba025a1c2411fb84c74d19f81f859b9004ecd6c9417a4ce9d3fd1cb07da24cb92de2164537faf8cb6
+  metadata.gz: 0f2ad1f0efb0712fac07ffb62198695be09a27a0f2e3346c97532331973cae21cbede3002c8e0c935945cccdcbbf6027e6297f2621fe8add53f4b47b3fbb2e18
+  data.tar.gz: 31e5c848db45f20b5a5358b602ee08ad285ba44bb40f20265792a8331067ba51c1cba4e78b6f260264a6bd480d95447e10031005e3b8d18a336ad93f467695c7

data/bin/miga

@@ -4,6 +4,8 @@
 # @license Artistic-2.0
 
 $LOAD_PATH.push File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.push File.expand_path('../../lib', File.realpath(__FILE__))
+
 require 'miga'
 require 'miga/cli'
 

data/lib/miga/cli/action/daemon.rb

@@ -73,6 +73,7 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
   end
 
   def perform
+    cli.operation or raise 'Please specify a daemon operation'
     p = cli.load_project
     d = MiGA::Daemon.new(p, cli[:json])
     dopts = %i[latency maxjobs nodelist ppn shutdown_when_done]

data/lib/miga/cli/action/doctor/base.rb

@@ -1,5 +1,5 @@
 require 'miga/cli/action'
-require 'sqlite3'
+require 'miga/sqlite'
 
 class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
 end
@@ -10,9 +10,7 @@ module MiGA::Cli::Action::Doctor::Base
   # tables saving +metric+ (:ani or :aai) and call +blk+ if the
   # file is corrupt or doesn't contain the expected structure
   def check_sqlite3_database(db_file, metric, &blk)
-    SQLite3::Database.new(db_file) do |conn|
-      conn.execute("select count(*) from #{metric}").first
-    end
+    MiGA::SQLite.new(db_file).run("select count(*) from #{metric}")
   rescue SQLite3::SQLException, SQLite3::CorruptException
     blk.call
   end
@@ -107,11 +105,7 @@ module MiGA::Cli::Action::Doctor::Base
     dist = dataset.result(:distances) or return
     path = dist.file_path(:aai_db) or return
 
-    o = []
-    SQLite3::Database.new(path) do |conn|
-      o = conn.execute('select seq2 from aai').map(&:first)
-    end
-    o
+    MiGA::SQLite.new(path).run('select seq2 from aai').map(&:first)
   end
 
   ##
@@ -120,34 +114,20 @@ module MiGA::Cli::Action::Doctor::Base
   def save_bidirectional(a, b)
     each_database_file(a) do |db_file, metric, result|
       data = nil
-      SQLite3::Database.new(db_file) do |conn|
-        data =
-          conn.execute(
-            "select seq1, seq2, #{metric}, sd, n, omega " +
-            "from #{metric} where seq2 = ? limit 1", b.name
-          ).first
-      end
+      data = MiGA::SQLite.new(db_file).run(
+        "select seq1, seq2, #{metric}, sd, n, omega " +
+        "from #{metric} where seq2 = ? limit 1", b.name
+      ).first
       next if data.nil? || data.empty?
 
       db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
       next unless File.exist?(db_file_b)
 
       data[0], data[1] = data[1], data[0]
-      SQLite3::Database.new(db_file_b) do |conn|
-        attempts = 0
-        begin
-          attempts += 1
-          conn.execute(
-            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-            "values(?, ?, ?, ?, ?, ?)", data
-          )
-        rescue SQLite3::BusyException => e
-          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
-
-          sleep(1)
-          retry
-        end
-      end
+      MiGA::SQLite.new(db_file_b).run(
+        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+        "values(?, ?, ?, ?, ?, ?)", data
+      )
     end
   end
 end

data/lib/miga/cli/action/run.rb

@@ -59,7 +59,7 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
            "MIGA=#{miga.shellescape}", "CORES=#{cli[:thr]}"]
     obj = cli.load_project_or_dataset
     klass = obj.class
-    virtual_task = [:p, :d].include?(cli[:result])
+    virtual_task = %i[p d maintenance].include?(cli[:result])
     cmd << "DATASET=#{obj.name.shellescape}" if obj.is_a? MiGA::Dataset
     if klass.RESULT_DIRS[cli[:result]].nil? and not virtual_task
       raise "Unsupported #{klass.to_s.sub(/.*::/, '')} result: #{cli[:result]}."

data/lib/miga/cli/action/wf.rb

@@ -184,7 +184,7 @@ module MiGA::Cli::Action::Wf
   def call_cli(cmd)
     cmd << '-v' if cli[:verbose]
     MiGA::MiGA.DEBUG "Cli::Action::Wf.call_cli #{cmd}"
-    MiGA::Cli.new(cmd.map(&:to_s)).launch
+    MiGA::Cli.new(cmd.map(&:to_s)).launch(true)
   end
 
   def run_daemon

data/lib/miga/common.rb

@@ -74,10 +74,13 @@ class MiGA::MiGA
     @_advance_time[:n] = n
 
     # Report
-    adv_vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
     adv =
-      total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-        ('%.1f%% (%s/%s)' % adv_vals)
+      if total.nil?
+        (n == 0 ? '' : num_suffix(n, bin))
+      else
+        vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
+        ('%.1f%% (%s/%s)' % vals)
+      end
     left =
       if @_advance_time[:avg].nil?
         ''

data/lib/miga/common/with_daemon.rb

@@ -158,7 +158,7 @@ module MiGA::Common::WithDaemon
   # Returns the status of the daemon with +opts+
   def status(opts = [], wait = true)
     if active?
-      say "Running with pid #{File.read(pid_file)}"
+      say "Running with pid #{File.size?(pid_file) ? File.read(pid_file) : '?'}"
     else
       say 'Not running'
     end

data/lib/miga/daemon.rb

@@ -73,7 +73,7 @@ class MiGA::Daemon < MiGA::MiGA
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
     miga_say "Saving log to: #{output_file}" unless show_log?
-    recalculate_status!
+    queue_maintenance
     load_status
     say 'Configuration options:'
     say @runopts.to_s
@@ -92,8 +92,7 @@ class MiGA::Daemon < MiGA::MiGA
     flush!
     if (loop_i % 12).zero?
       purge!
-      # TEMPORARILY DISABLED:
-      # recalculate_status!
+      queue_maintenance
     end
     save_status
     sleep(latency)
@@ -101,9 +100,13 @@ class MiGA::Daemon < MiGA::MiGA
     true
   end
 
-  def recalculate_status!
-    say 'Recalculating status for all datasets'
-    project.each_dataset(&:recalculate_status)
+  ##
+  # Queue maintenance tasks as an analysis job
+  def queue_maintenance
+    return if bypass_maintenance?
+
+    say 'Queueing maintenance tasks'
+    queue_job(:maintenance)
   end
 
   ##
@@ -179,6 +182,7 @@ class MiGA::Daemon < MiGA::MiGA
     unless show_log?
       n = project.dataset_names.count
       k = jobs_to_run.size + jobs_running.size
+      k -= 1 unless get_job(:maintenance).nil?
       advance('Datasets:', n - k, n, false)
       miga_say if k == 0
     end
@@ -271,6 +275,11 @@ class MiGA::Daemon < MiGA::MiGA
     # Avoid single datasets hogging resources
     @jobs_to_run.rotate! rand(jobs_to_run.size)
 
+    # Prioritize: Project-wide > MiGA Online queries > Other datasets
+    @jobs_to_run.sort_by! do |job|
+      job[:ds].nil? ? 1 : job[:ds_name] =~ /^qG_/ ? 2 : 3
+    end
+
     # Launch as many +jobs_to_run+ as possible
     while (hostk = next_host)
       break if jobs_to_run.empty?
@@ -309,11 +318,13 @@ class MiGA::Daemon < MiGA::MiGA
       job[:hostk] = hostk
       job[:cmd] = job[:cmd].miga_variables(host: nodelist[hostk])
       job[:pid] = spawn job[:cmd]
-      Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
+      MiGA::MiGA.DEBUG "Detaching PID: #{job[:pid]}"
+      Process.detach(job[:pid]) unless [nil, '', 0].include?(job[:pid])
     when 'bash'
       # Local job
       job[:pid] = spawn job[:cmd]
-      Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
+      MiGA::MiGA.DEBUG "Detaching PID: #{job[:pid]}"
+      Process.detach(job[:pid]) unless [nil, '', 0].include?(job[:pid])
     else
       # Schedule cluster job (qsub, msub, slurm)
       job[:pid] = `#{job[:cmd]}`.chomp

data/lib/miga/daemon/base.rb

@@ -17,7 +17,7 @@ module MiGA::Daemon::Base
         if !force && v == 0 && k != :verbosity
           raise "Daemon's #{k} cannot be set to zero"
         end
-      when :shutdown_when_done, :show_log
+      when :shutdown_when_done, :show_log, :bypass_maintenance
         v = !!v
       when :nodelist
         if v =~ /^\$/
@@ -63,6 +63,12 @@ module MiGA::Daemon::Base
     !!runopts(:shutdown_when_done)
   end
 
+  ##
+  # Should the daemon ignore regular maintenance steps?
+  def bypass_maintenance?
+    !!runopts(:bypass_maintenance)
+  end
+
   ##
   # Returns the level of verbosity for the daemon as an Integer, or 1 if unset.
   # Verbosity levels are:

data/lib/miga/dataset.rb

@@ -7,7 +7,11 @@ require 'miga/metadata'
 require 'miga/dataset/result'
 require 'miga/dataset/status'
 require 'miga/dataset/hooks'
-require 'sqlite3'
+
+# This library is only required by +#closest_relatives+, so it is now
+# being loaded on call instead to allow most of miga-base to work without
+# issue in systems with problematic SQLite3 installations.
+# require 'miga/sqlite'
 
 ##
 # Dataset representation in MiGA
@@ -190,8 +194,8 @@ class MiGA::Dataset < MiGA::MiGA
     r = result(ref_project ? :taxonomy : :distances)
     return nil if r.nil?
 
-    db = SQLite3::Database.new(r.file_path(:aai_db))
-    db.execute(
+    require 'miga/sqlite'
+    MiGA::SQLite.new(r.file_path(:aai_db)).run(
       'SELECT seq2, aai FROM aai WHERE seq2 != ? ' \
       'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many]
     )

data/lib/miga/dataset/result.rb

@@ -1,8 +1,12 @@
-require 'sqlite3'
 require 'miga/result'
 require 'miga/dataset/base'
 require 'miga/common/with_result'
 
+# This library is only required by +#cleanup_distances!+, so it is now
+# being loaded on call instead to allow most of miga-base to work without
+# issue in systems with problematic SQLite3 installations.
+# require 'miga/sqlite'
+
 ##
 # Helper module including specific functions to add dataset results
 module MiGA::Dataset::Result
@@ -144,20 +148,21 @@ module MiGA::Dataset::Result
   # the project as reference datasets.
   def cleanup_distances!
     r = get_result(:distances)
-    ref = project.datasets.select(&:ref?).select(&:active?).map(&:name)
     return if r.nil?
 
+    require 'miga/sqlite'
+    ref = project.datasets.select(&:ref?).select(&:active?).map(&:name)
     %i[haai_db aai_db ani_db].each do |db_type|
       db = r.file_path(db_type)
       next if db.nil? || !File.size?(db)
 
-      sqlite_db = SQLite3::Database.new db
+      sqlite_db = MiGA::SQLite.new(db)
       table = db_type[-6..-4]
-      val = sqlite_db.execute "select seq2 from #{table}"
+      val = sqlite_db.run("select seq2 from #{table}")
       next if val.empty?
 
       (val.map(&:first) - ref).each do |extra|
-        sqlite_db.execute "delete from #{table} where seq2=?", extra
+        sqlite_db.run("delete from #{table} where seq2=?", extra)
       end
     end
   end

data/lib/miga/remote_dataset.rb

@@ -156,10 +156,12 @@ class MiGA::RemoteDataset < MiGA::MiGA
     return nil unless metadata[:ncbi_asm]
 
     ncbi_asm_id = self.class.ncbi_asm_acc2id metadata[:ncbi_asm]
-    doc = MiGA::Json.parse(
-      self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json),
-      symbolize: false, contents: true
-    )
+    txt = nil
+    3.times do
+      txt = self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json)
+      txt.empty? ? sleep(1) : break
+    end
+    doc = MiGA::Json.parse(txt, symbolize: false, contents: true)
     @_ncbi_asm_json_doc = doc['result'][ doc['result']['uids'].first ]
   end
 

data/lib/miga/sqlite.rb

@@ -0,0 +1,49 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'sqlite3'
+
+##
+# SQLite3 wrapper for MiGA.
+class MiGA::SQLite < MiGA::MiGA
+  class << self
+    ##
+    # Default parsing options. Supported +opts+ keys:
+    # - +:busy_attempts+: Number of times to retry when database is busy
+    #   (default: 3)
+    def default_opts(opts = {})
+      opts[:busy_attempts] ||= 3
+      opts
+    end
+  end
+
+  ##
+  # Options hash
+  attr :opts
+
+  ##
+  # Database absolute path
+  attr :path
+
+  ##
+  # Create MiGA::SQLite with database in +path+ (without opening a connection)
+  # and options +opts+ (see +.default_opts+)
+  def initialize(path, opts = {})
+    @opts = MiGA::SQLite.default_opts(opts)
+    @path = File.absolute_path(path)
+  end
+
+  ##
+  # Executes +cmd+ and returns the result
+  def run(*cmd)
+    busy_attempts ||= 0
+    conn = SQLite3::Database.new(path)
+    conn.execute(*cmd)
+  rescue SQLite3::BusyException => e
+    busy_attempts += 1
+    raise "Database busy #{path}: #{e.message}" if busy_attempts >= 3
+
+    sleep(1)
+    retry
+  end
+end

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 16, 8]
+  VERSION = [0.7, 18, 4]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 11, 22)
+  VERSION_DATE = Date.new(2021, 1, 1)
 
   ##
   # Reference of MiGA.

data/scripts/maintenance.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
+SCRIPT="maintenance"
+# shellcheck source=scripts/miga.bash
+. "$MIGA/scripts/miga.bash" || exit 1
+
+miga doctor --only status -P "$PROJECT" -t "$CORES" -v
+

data/scripts/miga.bash

@@ -11,17 +11,17 @@ function exists { [[ -e "$1" ]] ; }
 function fx_exists { [[ $(type -t "$1") == "function" ]] ; }
 
 if [[ "$SCRIPT" != "d" && "$SCRIPT" != "p" ]] ; then
-  echo '############'
-  echo -n "Date: " ; miga date
-  echo "Hostname: $(hostname)"
-  echo "MiGA: $MIGA"
-  echo "Task: $SCRIPT"
-  echo "Project: $PROJECT"
+  echo ""
+  echo "######[ $SCRIPT ]######"
+  echo "# Date: $(miga date)"
+  echo "# Host: $(hostname)"
+  echo "# MiGA: $MIGA"
+  echo "# Project: $PROJECT"
   if [[ -n $DATASET ]] ; then
-    echo "Dataset: $DATASET"
+    echo "# Dataset: $DATASET"
     miga edit -P "$PROJECT" -D "$DATASET" -m "_step=$SCRIPT"
   fi
-  echo '------------'
+  echo '#------------'
 fi
 
 true

data/scripts/project_stats.bash

@@ -12,12 +12,15 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
+echo "# Doctor"
 miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
+echo "# Index taxonomy"
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active
 
 # Index metadata
+echo "# Index metadata"
 ruby -I "$MIGA/lib" \
   "$MIGA/utils/index_metadata.rb" "$PROJECT" "miga-project.metadata.db"
 

data/test/daemon_test.rb

@@ -93,7 +93,7 @@ class DaemonTest < Test::Unit::TestCase
       0 => /-{20}\n/,
       1 => /MiGA:#{p.name} launched/,
       2 => /-{20}\n/,
-      6 => /Probing running jobs\n/
+      8 => /Probing running jobs\n/
     }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
   ensure
     begin
@@ -160,8 +160,9 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d1.jobs_running.size)
     assert_equal(0, d1.jobs_to_run.size)
     capture_stderr { d1.in_loop }
+    # 3 dataset jobs + 1 maintenance job:
     assert_equal(1, d1.jobs_running.size)
-    assert_equal(2, d1.jobs_to_run.size)
+    assert_equal(3, d1.jobs_to_run.size)
   end
 
   def test_maxjobs_runopts
@@ -172,7 +173,7 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d1.jobs_to_run.size)
     capture_stderr { d1.in_loop }
     assert_equal(2, d1.jobs_running.size)
-    assert_equal(1, d1.jobs_to_run.size)
+    assert_equal(2, d1.jobs_to_run.size)
   end
 
   def test_load_status
@@ -232,6 +233,7 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_shutdown_when_done
+    daemon.runopts(:bypass_maintenance, true)
     daemon.runopts(:shutdown_when_done, true)
     out = capture_stderr { assert { !daemon.in_loop } }.string
     assert_match(/Nothing else to do/, out)
@@ -317,4 +319,20 @@ class DaemonTest < Test::Unit::TestCase
     out = capture_stderr { d1.in_loop }.string
     assert_match(/Daemon loop start/, out)
   end
+
+  def test_bypass_maintenance
+    # Default (run maintenance)
+    d = daemon(0)
+    d.runopts(:latency, 0, true)
+    capture_stderr { d.in_loop }
+    assert_equal(1, d.jobs_running.size)
+    assert_equal(:maintenance, d.jobs_running.first[:job])
+
+    # Bypassing maintenance
+    d = daemon(1)
+    d.runopts(:latency, 0, true)
+    d.runopts(:bypass_maintenance, true)
+    capture_stderr { d.in_loop }
+    assert_equal([], d.jobs_running)
+  end
 end

data/utils/cleanup-databases.rb

@@ -5,21 +5,17 @@ require 'miga'
 
 ARGV[1] or abort "Usage: #{$0} path/to/project threads"
 
-$stderr.puts 'Cleaning databases...'
 p = MiGA::Project.load(ARGV[0])
-ds_names = p.dataset_names
+dsn = p.dataset_names
 thr = ARGV[1].to_i
 
-pc = [0] + (1..100).map { |i| ds_names.size * i / 100 }
-$stderr.puts (('.' * 9 + '|') * 10) + ' 100%'
+m = MiGA::MiGA.new
+m.say 'Cleaning Databases'
 
 (0..thr - 1).each do |t|
   fork do
-    ds_names.each_with_index do |i, idx|
-      while t == 0 and idx + 1 > pc.first
-        $stderr.print '#'
-        pc.shift
-      end
+    dsn.each_with_index do |i, idx|
+      m.advance('Dataset:', dsn.size, idx + 1) if t == 0
       next unless (idx % thr) == t
 
       d = p.dataset(i)
@@ -30,4 +26,6 @@ $stderr.puts (('.' * 9 + '|') * 10) + ' 100%'
   end
 end
 Process.waitall
-$stderr.puts ' Done'
+m.advance('Dataset:', dsn.size, dsn.size)
+m.say
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.16.8
+  version: 0.7.18.4
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-22 00:00:00.000000000 Z
+date: 2021-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -196,6 +196,7 @@ files:
 - lib/miga/result/dates.rb
 - lib/miga/result/source.rb
 - lib/miga/result/stats.rb
+- lib/miga/sqlite.rb
 - lib/miga/tax_dist.rb
 - lib/miga/tax_index.rb
 - lib/miga/taxonomy.rb
@@ -211,6 +212,7 @@ files:
 - scripts/essential_genes.bash
 - scripts/haai_distances.bash
 - scripts/init.bash
+- scripts/maintenance.bash
 - scripts/miga.bash
 - scripts/mytaxa.bash
 - scripts/mytaxa_scan.bash