miga-base 0.7.16.7 → 0.7.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: affb99901724300ab46e2e7b1ef3f4e227f4288bef3a0634c8caa5f86b10b345
-  data.tar.gz: fc51e29cc954afd22ae6ab59c59393165a3ea64455807bac8e2300660f2c2673
+  metadata.gz: 78b020bda6120b5a284100b9544d7a488d385630f21b5d6d5e2bb9b365c832fb
+  data.tar.gz: d8332711fdf86308726700654ee8db91bb097f022a26e1240e62d107efa833a1
 SHA512:
-  metadata.gz: f8c9f1c7c307e408080ea80a4496eba48fe9d05220b6e636490958358c5a19da1f2ee86971c4b02b8200fcd8d5f01ec949b62dfbfce7b70b68da8ce940c7f5bf
-  data.tar.gz: cc4fed1bdef75754f4110dc768cc87b4766e6f7cf91166fdfa321657952066831b39b6da10aa598929a0155c93f555ba7ed0c0afe3dc118a92a7c954ce271551
+  metadata.gz: 8b766a40acf37193204e6369e07f2ecba1a67aa6141133654ab69f89b2ba9a63a959bd3f5b7fd21c56542eea7e3174cc342b05dc76b73ed27c6a2b273c546ca9
+  data.tar.gz: 6f8d791b9bf443a1eb43198625ebce48501ebaa3680be0c4cef55931b176fd2365407865e6e43cd549217ada6c67e5c86b156bd0816b80f91c388af812af810c

data/bin/miga

@@ -4,6 +4,8 @@
 # @license Artistic-2.0
 
 $LOAD_PATH.push File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.push File.expand_path('../../lib', File.realpath(__FILE__))
+
 require 'miga'
 require 'miga/cli'
 

data/lib/miga/cli/action/daemon.rb

@@ -73,6 +73,7 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
   end
 
   def perform
+    cli.operation or raise 'Please specify a daemon operation'
     p = cli.load_project
     d = MiGA::Daemon.new(p, cli[:json])
     dopts = %i[latency maxjobs nodelist ppn shutdown_when_done]

data/lib/miga/cli/action/doctor/base.rb

@@ -1,5 +1,5 @@
 require 'miga/cli/action'
-require 'sqlite3'
+require 'miga/sqlite'
 
 class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
 end
@@ -10,9 +10,7 @@ module MiGA::Cli::Action::Doctor::Base
   # tables saving +metric+ (:ani or :aai) and call +blk+ if the
   # file is corrupt or doesn't contain the expected structure
   def check_sqlite3_database(db_file, metric, &blk)
-    SQLite3::Database.new(db_file) do |conn|
-      conn.execute("select count(*) from #{metric}").first
-    end
+    MiGA::SQLite.new(db_file).run("select count(*) from #{metric}")
   rescue SQLite3::SQLException, SQLite3::CorruptException
     blk.call
   end
@@ -107,11 +105,7 @@ module MiGA::Cli::Action::Doctor::Base
     dist = dataset.result(:distances) or return
     path = dist.file_path(:aai_db) or return
 
-    o = []
-    SQLite3::Database.new(path) do |conn|
-      o = conn.execute('select seq2 from aai').map(&:first)
-    end
-    o
+    MiGA::SQLite.new(path).run('select seq2 from aai').map(&:first)
   end
 
   ##
@@ -120,34 +114,20 @@ module MiGA::Cli::Action::Doctor::Base
   def save_bidirectional(a, b)
     each_database_file(a) do |db_file, metric, result|
       data = nil
-      SQLite3::Database.new(db_file) do |conn|
-        data =
-          conn.execute(
-            "select seq1, seq2, #{metric}, sd, n, omega " +
-            "from #{metric} where seq2 = ? limit 1", b.name
-          ).first
-      end
+      data = MiGA::SQLite.new(db_file).run(
+        "select seq1, seq2, #{metric}, sd, n, omega " +
+        "from #{metric} where seq2 = ? limit 1", b.name
+      ).first
       next if data.nil? || data.empty?
 
       db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
       next unless File.exist?(db_file_b)
 
       data[0], data[1] = data[1], data[0]
-      SQLite3::Database.new(db_file_b) do |conn|
-        attempts = 0
-        begin
-          attempts += 1
-          conn.execute(
-            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-            "values(?, ?, ?, ?, ?, ?)", data
-          )
-        rescue SQLite3::BusyException => e
-          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
-
-          sleep(1)
-          retry
-        end
-      end
+      MiGA::SQLite.new(db_file_b).run(
+        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+        "values(?, ?, ?, ?, ?, ?)", data
+      )
     end
   end
 end

data/lib/miga/cli/action/run.rb

@@ -37,9 +37,9 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
       cli[:thr] ||= ENV['CORES'].to_i unless ENV['CORES'].nil?
       cli[:result] = File.basename(cli[:result].to_s, '.bash').to_sym
     end
-    cli[:project] = nil if cli[:project].empty?
-    cli[:dataset] = nil if cli[:dataset].empty?
-    cli[:result] = nil if cli[:result].empty?
+    %i[project dataset result].each do |i|
+      cli[i] = nil if cli[i].nil? || cli[i].empty?
+    end
 
     # Unset dataset if the requested result is for projects
     if (MiGA::Project.RESULT_DIRS.keys + [:p]).include? cli[:result]

data/lib/miga/cli/action/wf.rb

@@ -184,7 +184,7 @@ module MiGA::Cli::Action::Wf
   def call_cli(cmd)
     cmd << '-v' if cli[:verbose]
     MiGA::MiGA.DEBUG "Cli::Action::Wf.call_cli #{cmd}"
-    MiGA::Cli.new(cmd.map(&:to_s)).launch
+    MiGA::Cli.new(cmd.map(&:to_s)).launch(true)
   end
 
   def run_daemon

data/lib/miga/daemon.rb

@@ -73,7 +73,7 @@ class MiGA::Daemon < MiGA::MiGA
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
     miga_say "Saving log to: #{output_file}" unless show_log?
-    recalculate_status!
+    queue_maintenance
     load_status
     say 'Configuration options:'
     say @runopts.to_s
@@ -92,8 +92,7 @@ class MiGA::Daemon < MiGA::MiGA
     flush!
     if (loop_i % 12).zero?
       purge!
-      # TEMPORARILY DISABLED:
-      # recalculate_status!
+      queue_maintenance
     end
     save_status
     sleep(latency)
@@ -101,9 +100,13 @@ class MiGA::Daemon < MiGA::MiGA
     true
   end
 
-  def recalculate_status!
-    say 'Recalculating status for all datasets'
-    project.each_dataset(&:recalculate_status)
+  ##
+  # Queue maintenance tasks as an analysis job
+  def queue_maintenance
+    return if bypass_maintenance?
+
+    say 'Queueing maintenance tasks'
+    queue_job(:maintenance)
   end
 
   ##
@@ -271,6 +274,11 @@ class MiGA::Daemon < MiGA::MiGA
     # Avoid single datasets hogging resources
     @jobs_to_run.rotate! rand(jobs_to_run.size)
 
+    # Prioritize project-wide jobs
+    project_jobs = @jobs_to_run.select { |i| i[:ds].nil? }
+    @jobs_to_run.delete_if { |i| i[:ds].nil? }
+    @jobs_to_run.prepend(*project_jobs)
+
     # Launch as many +jobs_to_run+ as possible
     while (hostk = next_host)
       break if jobs_to_run.empty?
@@ -309,11 +317,13 @@ class MiGA::Daemon < MiGA::MiGA
       job[:hostk] = hostk
       job[:cmd] = job[:cmd].miga_variables(host: nodelist[hostk])
       job[:pid] = spawn job[:cmd]
-      Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
+      MiGA::MiGA.DEBUG "Detaching PID: #{job[:pid]}"
+      Process.detach(job[:pid]) unless [nil, '', 0].include?(job[:pid])
     when 'bash'
       # Local job
       job[:pid] = spawn job[:cmd]
-      Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
+      MiGA::MiGA.DEBUG "Detaching PID: #{job[:pid]}"
+      Process.detach(job[:pid]) unless [nil, '', 0].include?(job[:pid])
     else
       # Schedule cluster job (qsub, msub, slurm)
       job[:pid] = `#{job[:cmd]}`.chomp

data/lib/miga/daemon/base.rb

@@ -17,7 +17,7 @@ module MiGA::Daemon::Base
         if !force && v == 0 && k != :verbosity
           raise "Daemon's #{k} cannot be set to zero"
         end
-      when :shutdown_when_done, :show_log
+      when :shutdown_when_done, :show_log, :bypass_maintenance
         v = !!v
       when :nodelist
         if v =~ /^\$/
@@ -63,6 +63,12 @@ module MiGA::Daemon::Base
     !!runopts(:shutdown_when_done)
   end
 
+  ##
+  # Should the daemon ignore regular maintenance steps?
+  def bypass_maintenance?
+    !!runopts(:bypass_maintenance)
+  end
+
   ##
   # Returns the level of verbosity for the daemon as an Integer, or 1 if unset.
   # Verbosity levels are:

data/lib/miga/dataset.rb

@@ -7,7 +7,7 @@ require 'miga/metadata'
 require 'miga/dataset/result'
 require 'miga/dataset/status'
 require 'miga/dataset/hooks'
-require 'sqlite3'
+require 'miga/sqlite'
 
 ##
 # Dataset representation in MiGA
@@ -190,8 +190,7 @@ class MiGA::Dataset < MiGA::MiGA
     r = result(ref_project ? :taxonomy : :distances)
     return nil if r.nil?
 
-    db = SQLite3::Database.new(r.file_path(:aai_db))
-    db.execute(
+    MiGA::SQLite.new(r.file_path(:aai_db)).run(
       'SELECT seq2, aai FROM aai WHERE seq2 != ? ' \
       'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many]
     )

data/lib/miga/dataset/result.rb

@@ -1,4 +1,4 @@
-require 'sqlite3'
+require 'miga/sqlite'
 require 'miga/result'
 require 'miga/dataset/base'
 require 'miga/common/with_result'
@@ -151,13 +151,13 @@ module MiGA::Dataset::Result
       db = r.file_path(db_type)
       next if db.nil? || !File.size?(db)
 
-      sqlite_db = SQLite3::Database.new db
+      sqlite_db = MiGA::SQLite.new(db)
       table = db_type[-6..-4]
-      val = sqlite_db.execute "select seq2 from #{table}"
+      val = sqlite_db.run("select seq2 from #{table}")
       next if val.empty?
 
       (val.map(&:first) - ref).each do |extra|
-        sqlite_db.execute "delete from #{table} where seq2=?", extra
+        sqlite_db.run("delete from #{table} where seq2=?", extra)
       end
     end
   end

data/lib/miga/remote_dataset.rb

@@ -156,10 +156,12 @@ class MiGA::RemoteDataset < MiGA::MiGA
     return nil unless metadata[:ncbi_asm]
 
     ncbi_asm_id = self.class.ncbi_asm_acc2id metadata[:ncbi_asm]
-    doc = MiGA::Json.parse(
-      self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json),
-      symbolize: false, contents: true
-    )
+    txt = nil
+    3.times do
+      txt = self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json)
+      txt.empty? ? sleep(1) : break
+    end
+    doc = MiGA::Json.parse(txt, symbolize: false, contents: true)
     @_ncbi_asm_json_doc = doc['result'][ doc['result']['uids'].first ]
   end
 

data/lib/miga/sqlite.rb

@@ -0,0 +1,49 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'sqlite3'
+
+##
+# SQLite3 wrapper for MiGA.
+class MiGA::SQLite < MiGA::MiGA
+  class << self
+    ##
+    # Default parsing options. Supported +opts+ keys:
+    # - +:busy_attempts+: Number of times to retry when database is busy
+    #   (default: 3)
+    def default_opts(opts = {})
+      opts[:busy_attempts] ||= 3
+      opts
+    end
+  end
+
+  ##
+  # Options hash
+  attr :opts
+
+  ##
+  # Database absolute path
+  attr :path
+
+  ##
+  # Create MiGA::SQLite with database in +path+ (without opening a connection)
+  # and options +opts+ (see +.default_opts+)
+  def initialize(path, opts = {})
+    @opts = MiGA::SQLite.default_opts(opts)
+    @path = File.absolute_path(path)
+  end
+
+  ##
+  # Executes +cmd+ and returns the result
+  def run(*cmd)
+    busy_attempts ||= 0
+    conn = SQLite3::Database.new(path)
+    conn.execute(*cmd)
+  rescue SQLite3::BusyException => e
+    busy_attempts += 1
+    raise "Database busy #{path}: #{e.message}" if busy_attempts >= 3
+
+    sleep(1)
+    retry
+  end
+end

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 16, 7]
+  VERSION = [0.7, 18, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 11, 22)
+  VERSION_DATE = Date.new(2020, 12, 27)
 
   ##
   # Reference of MiGA.

data/scripts/maintenance.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
+SCRIPT="maintenance"
+# shellcheck source=scripts/miga.bash
+. "$MIGA/scripts/miga.bash" || exit 1
+
+miga doctor --only status -P "$PROJECT" -t "$CORES" -v
+

data/scripts/project_stats.bash

@@ -12,12 +12,15 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
+echo "# Doctor"
 miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
+echo "# Index taxonomy"
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active
 
 # Index metadata
+echo "# Index metadata"
 ruby -I "$MIGA/lib" \
   "$MIGA/utils/index_metadata.rb" "$PROJECT" "miga-project.metadata.db"
 

data/test/daemon_test.rb

@@ -93,7 +93,7 @@ class DaemonTest < Test::Unit::TestCase
       0 => /-{20}\n/,
       1 => /MiGA:#{p.name} launched/,
       2 => /-{20}\n/,
-      6 => /Probing running jobs\n/
+      8 => /Probing running jobs\n/
     }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
   ensure
     begin
@@ -160,8 +160,9 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d1.jobs_running.size)
     assert_equal(0, d1.jobs_to_run.size)
     capture_stderr { d1.in_loop }
+    # 3 dataset jobs + 1 maintenance job:
     assert_equal(1, d1.jobs_running.size)
-    assert_equal(2, d1.jobs_to_run.size)
+    assert_equal(3, d1.jobs_to_run.size)
   end
 
   def test_maxjobs_runopts
@@ -172,7 +173,7 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d1.jobs_to_run.size)
     capture_stderr { d1.in_loop }
     assert_equal(2, d1.jobs_running.size)
-    assert_equal(1, d1.jobs_to_run.size)
+    assert_equal(2, d1.jobs_to_run.size)
   end
 
   def test_load_status
@@ -232,6 +233,7 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_shutdown_when_done
+    daemon.runopts(:bypass_maintenance, true)
     daemon.runopts(:shutdown_when_done, true)
     out = capture_stderr { assert { !daemon.in_loop } }.string
     assert_match(/Nothing else to do/, out)
@@ -317,4 +319,20 @@ class DaemonTest < Test::Unit::TestCase
     out = capture_stderr { d1.in_loop }.string
     assert_match(/Daemon loop start/, out)
   end
+
+  def test_bypass_maintenance
+    # Default (run maintenance)
+    d = daemon(0)
+    d.runopts(:latency, 0, true)
+    capture_stderr { d.in_loop }
+    assert_equal(1, d.jobs_running.size)
+    assert_equal(:maintenance, d.jobs_running.first[:job])
+
+    # Bypassing maintenance
+    d = daemon(1)
+    d.runopts(:latency, 0, true)
+    d.runopts(:bypass_maintenance, true)
+    capture_stderr { d.in_loop }
+    assert_equal([], d.jobs_running)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.16.7
+  version: 0.7.18.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-22 00:00:00.000000000 Z
+date: 2020-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -196,6 +196,7 @@ files:
 - lib/miga/result/dates.rb
 - lib/miga/result/source.rb
 - lib/miga/result/stats.rb
+- lib/miga/sqlite.rb
 - lib/miga/tax_dist.rb
 - lib/miga/tax_index.rb
 - lib/miga/taxonomy.rb
@@ -211,6 +212,7 @@ files:
 - scripts/essential_genes.bash
 - scripts/haai_distances.bash
 - scripts/init.bash
+- scripts/maintenance.bash
 - scripts/miga.bash
 - scripts/mytaxa.bash
 - scripts/mytaxa_scan.bash