miga-base 0.7.20.0 → 0.7.22.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de9486ffe4bff4fd61295c004f36af331ce5a50645f4aa706076a38d3523ee38
-  data.tar.gz: 60ded180b92cfb27729a4844c00713b4e78bf08b6a1f8ebf30f6bdccd24eb769
+  metadata.gz: 6c0bdcbce7b3490a9b011ab551789b11a48169404f92ef6e61aa8e1f3a05d998
+  data.tar.gz: ab789c8da45a841561fc6b1d91717907cd54fc52934607997e895d27e5bdf636
 SHA512:
-  metadata.gz: 7a5e89cf299d72c2b3fb9a267923caef0ecfc399cabfec71011e9ebc3c37d874cb8a93e16d169db7c3bbda1f54d3ab5863ba889341295339b69673d3457edef7
-  data.tar.gz: 4ac7cf2785e42afd17f19749632e1deda80cecfa10103f64d4bc366a29e4a8acf9ead5e8a412ae1b11ba966c6dc9501017cb7dbee2572ebcbc835e9db0341a7b
+  metadata.gz: 8c8177696eb041e7628810371b1fb347c831aaf2be5f74b0861b789520f656c8c275505b01c017802afb31439c2e7e20d940274e4493f77bf09b23e11703fadc
+  data.tar.gz: ae768cd8ff0037e109d9ab6e9a6bce448e9cdf56bc763b6e5a6571204ae33d7ffd26194fa47ae2380d028bf3fb37533f7a620c3f6cd57ac970c6bd7e432d4dcb

data/lib/miga/cli/action/add_result.rb

@@ -20,6 +20,6 @@ class MiGA::Cli::Action::AddResult < MiGA::Cli::Action
     obj = cli.load_project_or_dataset
     cli.say "Registering result: #{cli[:result]}"
     r = obj.add_result(cli[:result], true, force: cli[:force])
-    raise "Cannot add result, incomplete expected files." if r.nil?
+    raise 'Cannot add result, incomplete expected files' if r.nil?
   end
 end

data/lib/miga/cli/action/next_step.rb

@@ -11,16 +11,8 @@ class MiGA::Cli::Action::NextStep < MiGA::Cli::Action
   end
 
   def perform
-    p = cli.load_project
-    n = nil
-    if cli[:dataset].nil?
-      n = p.next_distances(false)
-      n ||= p.next_inclade(false)
-    else
-      d = cli.load_dataset
-      n = d.next_preprocessing if d.is_active?
-    end
-    n ||= '?'
-    cli.puts n
+    obj = cli.load_project_or_dataset
+    n = obj.next_task
+    cli.puts(n || '?')
   end
 end

data/lib/miga/common/with_result.rb

@@ -1,6 +1,8 @@
 ##
 # Helper module including specific functions to handle objects that
-# have results.
+# have results. The class including this module must implement methods
+# +.RESULT_DIRS+, +#ignore_task?+, +#metadata+, +#project+,
+# and +#inactivate!+.
 module MiGA::Common::WithResult
   ##
   # Result directories as a Hash
@@ -68,4 +70,40 @@ module MiGA::Common::WithResult
   def get_result(task)
     add_result(task, false)
   end
+
+  ##
+  # Get the next task from +tasks+, saving intermediate results if +save+.
+  # If +tasks+ is +nil+ (default), it uses the entire list of tasks.
+  # Returns a Symbol.
+  def next_task(tasks = nil, save = false)
+    tasks ||= result_dirs.keys
+    tasks.find do |t|
+      if ignore_task?(t)
+        # Do not run if this step is to be ignored
+        false
+      else
+        res = add_result(t, save)
+        if res.nil?
+          # Run if the step has not been calculated,
+          # unless too many attempts were already made
+          if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
+            inactivate! "Too many errors in step #{t}"
+            false
+          else
+            true
+          end
+        else
+          # Run if the step is ready but has to be recalculated
+          res.recalculate? ? true : false
+        end
+      end
+    end
+  end
+
+  ##
+  # Mark all results for recalculation
+  def recalculate_tasks(reason = nil)
+    each_result { |res| res.recalculate!(reason).save }
+  end
+
 end

data/lib/miga/daemon.rb

@@ -103,7 +103,7 @@ class MiGA::Daemon < MiGA::MiGA
   ##
   # Queue maintenance tasks as an analysis job
   def queue_maintenance
-    return if bypass_maintenance?
+    return if bypass_maintenance? || shutdown_when_done?
 
     say 'Queueing maintenance tasks'
     queue_job(:maintenance)
@@ -214,29 +214,38 @@ class MiGA::Daemon < MiGA::MiGA
     return nil unless get_job(job, ds).nil?
 
     ds_name = (ds.nil? ? 'miga-project' : ds.name)
-    say 'Queueing %s:%s' % [ds_name, job]
+    task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
+    to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
+    say 'Queueing %s:%s' % [to_run[:ds_name], to_run[:job]]
+    @jobs_to_run << to_run
+  end
+
+  ##
+  # Construct the command for the given job definition with current
+  # daemon settings
+  def job_cmd(to_run)
     vars = {
       'PROJECT' => project.path,
       'RUNTYPE' => runopts(:type),
       'CORES' => ppn,
       'MIGA' => MiGA::MiGA.root_path
     }
-    vars['DATASET'] = ds.name unless ds.nil?
-    log_dir = File.expand_path("daemon/#{job}", project.path)
-    Dir.mkdir(log_dir) unless Dir.exist? log_dir
-    task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
-    to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
-    to_run[:cmd] = runopts(:cmd).miga_variables(
-      script: MiGA::MiGA.script_path(job, miga: vars['MIGA'], project: project),
+    vars['DATASET'] = to_run[:ds].name unless to_run[:ds].nil?
+    log_dir = File.expand_path("daemon/#{to_run[:job]}", project.path)
+    FileUtils.mkdir_p(log_dir)
+    var_hsh = {
+      script: MiGA::MiGA.script_path(
+                to_run[:job], miga: vars['MIGA'], project: project
+              ),
       vars: vars.map do |k, v|
               runopts(:var).miga_variables(key: k, value: v)
             end.join(runopts(:varsep)),
       cpus: ppn,
-      log: File.expand_path("#{ds_name}.log", log_dir),
-      task_name: task_name,
-      miga: File.expand_path('bin/miga', MiGA::MiGA.root_path).shellescape
-    )
-    @jobs_to_run << to_run
+      log: File.join(log_dir, "#{to_run[:ds_name]}.log"),
+      task_name: to_run[:task_name],
+      miga: File.join(MiGA::MiGA.root_path, 'bin/miga').shellescape
+    }
+    runopts(:cmd).miga_variables(var_hsh)
   end
 
   ##
@@ -312,6 +321,7 @@ class MiGA::Daemon < MiGA::MiGA
   # Launch the job described by Hash +job+ to +hostk+-th host
   def launch_job(job, hostk = nil)
     # Execute job
+    job[:cmd] = job_cmd(job)
     case runopts(:type)
     when 'ssh'
       # Remote job

data/lib/miga/daemon/base.rb

@@ -93,7 +93,7 @@ module MiGA::Daemon::Base
   ##
   # Display log instead of the progress summary
   def show_log!
-    @show_log = true
+    @runopts[:show_log] = true
   end
 
   ##

data/lib/miga/dataset.rb

@@ -107,6 +107,7 @@ class MiGA::Dataset < MiGA::MiGA
     metadata[:warn] = "Inactive: #{reason}" unless reason.nil?
     metadata[:inactive] = true
     metadata.save
+    project.recalculate_tasks('Reference dataset inactivated') if ref?
     pull_hook :on_inactivate
   end
 
@@ -116,6 +117,7 @@ class MiGA::Dataset < MiGA::MiGA
     metadata[:inactive] = nil
     metadata[:warn] = nil if metadata[:warn] && metadata[:warn] =~ /^Inactive: /
     metadata.save
+    project.recalculate_tasks('Reference dataset activated') if ref?
     pull_hook :on_activate
   end
 

data/lib/miga/dataset/result.rb

@@ -77,19 +77,8 @@ module MiGA::Dataset::Result
   # Returns the key symbol of the next task that needs to be executed or nil.
   # Passes +save+ to #add_result.
   def next_preprocessing(save = false)
-    first = first_preprocessing(save) or return nil
-    @@PREPROCESSING_TASKS[@@PREPROCESSING_TASKS.index(first)..-1].find do |t|
-      if ignore_task? t
-        false
-      elsif add_result(t, save).nil?
-        if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
-          inactivate! "Too many errors in step #{t}"
-          false
-        else
-          true
-        end
-      end
-    end
+    first_preprocessing(save) if save
+    next_task(nil, save)
   end
 
   ##

data/lib/miga/project.rb

@@ -97,13 +97,28 @@ class MiGA::Project < MiGA::MiGA
 
   ##
   # Is this a clade project?
-  def is_clade?
+  def clade?
     type == :clade
   end
 
+  ##
+  # Same as active? For backward compatibility
+  alias is_clade? clade?
+
   ##
   # Is this a project for multi-organism datasets?
-  def is_multi?
+  def multi?
     @@KNOWN_TYPES[type][:multi]
   end
+
+  ##
+  # Same as multi? For backward compatibility
+  alias is_multi? multi?
+
+  ##
+  # Is this project active? Currently a dummy function, returns
+  # always true.
+  def active?
+    true
+  end
 end

data/lib/miga/project/base.rb

@@ -50,14 +50,11 @@ module MiGA::Project::Base
     07.annotation/01.function/01.essential
     07.annotation/01.function/02.ssu
     07.annotation/02.taxonomy/01.mytaxa
-    07.annotation/03.qa 07.annotation/03.qa/01.checkm
-    07.annotation/03.qa/02.mytaxa_scan
+    07.annotation/03.qa 07.annotation/03.qa/02.mytaxa_scan
     08.mapping 08.mapping/01.read-ctg 08.mapping/02.read-gene
     09.distances 09.distances/01.haai 09.distances/02.aai
     09.distances/03.ani 09.distances/04.ssu 09.distances/05.taxonomy
     10.clades 10.clades/01.find 10.clades/02.ani 10.clades/03.ogs
-    10.clades/04.phylogeny 10.clades/04.phylogeny/01.essential
-    10.clades/04.phylogeny/02.core 10.clades/05.metadata
     90.stats
   ]
 

data/lib/miga/project/dataset.rb

@@ -48,10 +48,11 @@ module MiGA::Project::Dataset
   # Add dataset identified by +name+ and return MiGA::Dataset.
   def add_dataset(name)
     unless metadata[:datasets].include? name
-      MiGA::Dataset.new(self, name)
+      d = MiGA::Dataset.new(self, name)
       @metadata[:datasets] << name
       @dataset_names_hash = nil # Ensure loading even if +do_not_save+ is true
       save
+      recalculate_tasks('New reference dataset added') if d.ref? && d.active?
       pull_hook(:on_add_dataset, name)
     end
     dataset(name)
@@ -65,6 +66,7 @@ module MiGA::Project::Dataset
 
     self.metadata[:datasets].delete(name)
     save
+    recalculate_tasks('Reference dataset unlinked') if d.ref? && d.active?
     pull_hook(:on_unlink_dataset, name)
     d
   end

data/lib/miga/project/result.rb

@@ -23,6 +23,11 @@ module MiGA::Project::Result
     self
   end
 
+  ##
+  # Do nothing, only to comply with MiGA::Common::WithResult
+  def inactivate!(reason = nil)
+  end
+
   ##
   # Is this +task+ to be bypassed?
   def ignore_task?(task)
@@ -45,17 +50,6 @@ module MiGA::Project::Result
     next_task(@@INCLADE_TASKS, save)
   end
 
-  ##
-  # Get the next task from +tasks+, saving intermediate results if +save+.
-  # If +tasks+ is +nil+ (default), it uses the entire list of tasks.
-  # Returns a Symbol.
-  def next_task(tasks = nil, save = true)
-    tasks ||= @@DISTANCE_TASKS + @@INCLADE_TASKS
-    tasks.find do |t|
-      ignore_task?(t) ? false : add_result(t, save).nil?
-    end
-  end
-
   private
 
   ##

data/lib/miga/remote_dataset.rb

@@ -162,6 +162,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
       txt.empty? ? sleep(1) : break
     end
     doc = MiGA::Json.parse(txt, symbolize: false, contents: true)
+    return if doc.nil? || doc['result'].nil? || doc['result'].empty?
+
     @_ncbi_asm_json_doc = doc['result'][ doc['result']['uids'].first ]
   end
 

data/lib/miga/result.rb

@@ -59,9 +59,25 @@ class MiGA::Result < MiGA::MiGA
   end
 
   ##
-  # Register the result as cleaned
+  # Register the result as cleaned, returns self
   def clean!
     self[:clean] = true
+    self
+  end
+
+  ##
+  # Is the result marked to be recalculated? Returns Boolean
+  def recalculate?
+    !!self[:recalculate]
+  end
+
+  ##
+  # Mark the result to be recalculated, returns self
+  def recalculate!(reason = nil)
+    self[:recalculate] = true
+    self[:recalculate_why] = reason
+    self[:recalculate_when] = Time.now.to_s
+    self
   end
 
   ##

data/lib/miga/tax_index.rb

@@ -17,7 +17,7 @@ class MiGA::TaxIndex < MiGA::MiGA
   ##
   # Initialize an empty MiGA::TaxIndex
   def initialize
-    @root = MiGA::TaxIndexTaxon.new :root, 'biota'
+    @root = MiGA::TaxIndexTaxon.new(:root, 'biota')
     @datasets = []
   end
 
@@ -37,21 +37,22 @@ class MiGA::TaxIndex < MiGA::MiGA
   end
 
   ##
-  # Finds all the taxa in the collection at the +rank+ taxonomic rank.
+  # Finds all the taxa in the collection at the +rank+ taxonomic rank
   def taxa_by_rank(rank)
     rank = MiGA::Taxonomy.normalize_rank(rank)
     taxa = [@root]
     select = []
     loop do
       new_taxa = []
-      taxa.map { |tx| tx.children }.flatten.each do |ch|
+      taxa.map(&:children).flatten.each do |ch|
         if ch.rank == rank
           select << ch
-        elsif not ch.children.empty?
+        elsif !ch.children.empty?
           new_taxa << ch
         end
       end
-      break if new_taxa.empty?
+      taxa = new_taxa
+      break if taxa.empty?
     end
     select
   end
@@ -60,7 +61,8 @@ class MiGA::TaxIndex < MiGA::MiGA
   # Generate JSON String for the index.
   def to_json
     MiGA::Json.generate(
-      { root: root.to_hash, datasets: datasets.map { |d| d.name } }
+      root: root.to_hash,
+      datasets: datasets.map(&:name)
     )
   end
 
@@ -119,27 +121,33 @@ class MiGA::TaxIndexTaxon < MiGA::MiGA
   ##
   # Get the number of datasets in the taxon (including children).
   def datasets_count
-    children.map { |it| it.datasets_count }.reduce(datasets.size, :+)
+    children.map(&:datasets_count).reduce(datasets.size, :+)
   end
 
   ##
   # Get all the datasets in the taxon (including children).
   def all_datasets
-    children.map { |it| it.datasets }.reduce(datasets, :+)
+    children.map(&:datasets).reduce(datasets, :+)
   end
 
   ##
   # JSON String of the taxon.
   def to_json(*a)
-    { str: tax_str, datasets: datasets.map { |d| d.name },
-      children: children }.to_json(a)
+    {
+      str: tax_str,
+      datasets: datasets.map(&:name),
+      children: children
+    }.to_json(a)
   end
 
   ##
   # Hash representation of the taxon.
   def to_hash
-    { str: tax_str, datasets: datasets.map { |d| d.name },
-      children: children.map { |it| it.to_hash } }
+    {
+      str: tax_str,
+      datasets: datasets.map(&:name),
+      children: children.map(&:to_hash)
+    }
   end
 
   ##

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 20, 0]
+  VERSION = [0.7, 22, 2]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 1, 7)
+  VERSION_DATE = Date.new(2021, 1, 30)
 
   ##
   # Reference of MiGA.

data/scripts/aai_distances.bash

@@ -19,7 +19,7 @@ rm -f miga-project.txt
   for i in $DS ; do
     echo "SELECT CASE WHEN omega!=0 THEN 'AAI' ELSE 'hAAI_AAI' END," \
       " seq1, seq2, aai, sd, n, omega from aai;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/scripts/ani_distances.bash

@@ -18,7 +18,7 @@ rm -f miga-project.txt
   echo "metric a b value sd n omega" | tr " " "\\t"
   for i in $DS ; do
     echo "SELECT 'ANI', seq1, seq2, ani, sd, n, omega from ani ;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/scripts/haai_distances.bash

@@ -22,7 +22,7 @@ rm -f miga-project.txt
   echo "metric a b value sd n omega" | tr " " "\\t"
   for i in $DS ; do
     echo "SELECT 'hAAI', seq1, seq2, aai, sd, n, omega from aai ;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/test/common_test.rb

@@ -72,4 +72,34 @@ class CommonTest < Test::Unit::TestCase
     assert_not_predicate('C3-PO', :miga_name?)
     assert_equal("123\n1\n", '1231'.wrap_width(3))
   end
+
+  def test_advance
+    m = MiGA::MiGA.new
+
+    # Check advance when missing total
+    o = capture_stderr { m.advance('x', 0) }.string
+    assert_match(/\] x *\r/, o)
+
+    # Initialize advance
+    o = capture_stderr { m.advance('x', 0, 10) }.string
+    assert_match(/\] x 0\.0% \(0\/10\) *\r/, o)
+
+    # Insufficient data for prediction
+    sleep(1)
+    o = capture_stderr { m.advance('x', 1, 10) }.string
+    assert_match(/\] x 10\.0% \(1\/10\) *\r/, o)
+
+    # Predict time
+    sleep(1)
+    o = capture_stderr { m.advance('x', 2, 10) }.string
+    assert_match(/\] x 20\.0% \(2\/10\) [7-9]s left *\r/, o)
+  end
+
+  def test_num_suffix
+    m = MiGA::MiGA.new
+    assert_equal('12', m.num_suffix(12))
+    assert_equal('1.5K', m.num_suffix(1.5e3))
+    assert_equal('1.0M', m.num_suffix(1024**2 + 1, true))
+    assert_equal('1.1G', m.num_suffix(1024**3))
+  end
 end

data/test/daemon_test.rb

@@ -48,7 +48,10 @@ class DaemonTest < Test::Unit::TestCase
     end
     assert_match(/Queueing #{ds.name}:d/, out.string)
     assert_equal(1, d.jobs_to_run.size)
-    assert_equal('echo project0:d:ds1 >/dev/null', d.jobs_to_run.first[:cmd])
+    assert_equal(
+      'echo project0:d:ds1 >/dev/null',
+      d.job_cmd(d.jobs_to_run.first)
+    )
     assert_equal(d.jobs_to_run.first, d.get_job(:d, ds))
   end
 
@@ -335,4 +338,17 @@ class DaemonTest < Test::Unit::TestCase
     capture_stderr { d.in_loop }
     assert_equal([], d.jobs_running)
   end
+
+  def test_show_log
+    d = daemon
+    assert(d.show_log?)
+    d.runopts(:show_log, false)
+    assert(!d.show_log?)
+    d.show_log!
+    assert(d.show_log?)
+    assert_equal($stderr, d.logfh)
+    d.show_summary!
+    assert(!d.show_log?)
+    assert_not_equal($stderr, d.logfh)
+  end
 end

data/test/hook_test.rb

@@ -107,10 +107,10 @@ class HookTest < Test::Unit::TestCase
         File.join(project.path, 'data', '90.stats', "miga-project.#{ext}")
       )
     end
-    assert_equal(:project_stats, project.next_task(nil, false))
+    assert_equal(:project_stats, project.next_task)
     assert_equal(:test, $res)
     assert_equal(1, $counter)
-    assert_equal(:haai_distances, project.next_task)
+    assert_equal(:haai_distances, project.next_task(nil, true))
     assert_equal(:project_stats, $res)
     assert_equal(2, $counter)
   end

data/test/tax_index_test.rb

@@ -35,4 +35,30 @@ class TaxIndexTest < Test::Unit::TestCase
     ti = MiGA::TaxIndex.new
     assert_equal("root:biota: 0\n", ti.to_tab)
   end
+
+  def test_tax_index_taxon
+    taxon = MiGA::TaxIndexTaxon.new(:root, 'biota')
+    assert_empty(taxon.datasets)
+    assert_equal('root:biota', taxon.to_hash[:str])
+    assert_equal(
+      '{"str":"root:biota","datasets":[],"children":[]}',
+      taxon.to_json
+    )
+    assert_empty(taxon.all_datasets)
+    assert_equal(0, taxon.datasets_count)
+  end
+
+  def test_taxa_by_rank
+    initialize_miga_home
+    d = dataset
+    d.metadata[:tax] = MiGA::Taxonomy.new('k:Fantasia g:Unicornia')
+
+    # Empty
+    ti = MiGA::TaxIndex.new
+    assert_empty(ti.taxa_by_rank(:genus))
+
+    # Unicorn
+    ti << d
+    assert_equal('Unicornia', ti.taxa_by_rank(:genus).first.name)
+  end
 end

data/utils/cleanup-databases.rb

@@ -15,7 +15,7 @@ m.say 'Cleaning Databases'
 (0..thr - 1).each do |t|
   fork do
     dsn.each_with_index do |i, idx|
-      m.advance('Dataset:', dsn.size, idx + 1) if t == 0
+      m.advance('Dataset:', idx + 1, dsn.size) if t == 0
       next unless (idx % thr) == t
 
       d = p.dataset(i)

data/utils/subclade/runner.rb

@@ -15,7 +15,7 @@ class MiGA::SubcladeRunner
     @step = step.to_sym
     @home = File.join(
       File.join(project.path, 'data', '10.clades'),
-      @step == :clade_finding ? '01.find' : '02.ani'
+      @step == :clade_finding ? '01.find.running' : '02.ani.running'
     )
     @opts[:thr] ||= ENV.fetch('CORES') { 2 }.to_i
     @opts[:run_clades] = !!@project.metadata.data.fetch(:run_clades) { true }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.20.0
+  version: 0.7.22.2
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-07 00:00:00.000000000 Z
+date: 2021-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons