miga-base 0.6.3.0 → 0.6.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b225951f374bcd267560e5bd8234fb88bcd6b0c11b0561fb4b3b479af39c4b3
-  data.tar.gz: 9b32d40ea94ceb526fe0ba732c77fce978b0cba5decffd4e1c0d701594670dbd
+  metadata.gz: 9b94d26cedfe4ef7f4fd537dd668777c9ee3820eeb670a106e744250cdc72566
+  data.tar.gz: 37cfd446014fe396ef6ce15d5289e11118095349d400aeefe16f84985f77c27d
 SHA512:
-  metadata.gz: b7fe9b2cbb09b6612b762c7c9202b4b27ece7a0b6f4dd23eecee9bddc835c130f15a63772011106a1f0f1425e5445fa1541a8ebc81661ef341dcacec3ae22193
-  data.tar.gz: 501fa797aa6726ac5cdc6c043a3073d03a7ba3ed81d63ebac9ef989a76aa1947806c9052ab50c089567001ad1b1b92ad04cc05ea4ca33691de36c4dcb2e34b52
+  metadata.gz: 44e9dbe946ab889bb8bef0423544cec4d44894f9617ae28bf9ca33094ceaae16a662816110af42fc8a8941834bdf47f623eb6ec42afb3f0da405bb1267503d30
+  data.tar.gz: 30f712d1f2d9c540d571f66cd5b8b5d21e70540f85ec2b126f11949ea2029d935d5344594817c543d81ce26917c6e5d837798b185000d079c8c700a2ac6f13d8

data/lib/miga/cli/action/init/daemon_helper.rb

@@ -118,7 +118,7 @@ module MiGA::Cli::Action::Init::DaemonHelper
     v
   end
 
-  def configure_qsub_msub_daemon
+  def configure_qsub_msub_daemon(v)
     queue       = cli.ask_user('What queue should I use?', nil, nil, true)
     v[:latency] = cli.ask_user('How long should I sleep? (in secs)', '150').to_i
     v[:maxjobs] = cli.ask_user('How many jobs can I launch at once?', '300').to_i

data/lib/miga/cli/action/wf.rb

@@ -132,6 +132,7 @@ module MiGA::Cli::Action::Wf
     # Add datasets
     call_cli([
       'add',
+      '--ignore-dups',
       '-P', cli[:outdir],
       '-t', cli[:dataset_type],
       '-i', stage,

data/lib/miga/common/with_result.rb

@@ -0,0 +1,70 @@
+
+##
+# Helper module including specific functions to handle objects that
+# have results.
+module MiGA::Common::WithResult
+  ##
+  # Result directories as a Hash
+  def result_dirs
+    self.class.RESULT_DIRS
+  end
+
+  ##
+  # Look for the result with symbol key +task+ and register it in the object.
+  # If +save+ is false, it doesn't register the result, but it still returns a
+  # result if it already exists.
+  #
+  # The +opts+ hash controls result creation (if necessary).
+  # Supported values include:
+  # - +is_clean+: A Boolean indicating if the input files are clean
+  # - +force+: A Boolean indicating if the result must be re-indexed,
+  #   ignored unless +save = true+
+  #
+  # Returns MiGA::Result or nil
+  def add_result(task, save = true, opts = {})
+    task = task.to_sym
+    return nil if result_dirs[task].nil?
+    base = File.join(
+      project.path, "data/#{result_dirs[task]}/#{result_base}"
+    )
+    json = "#{base}.json"
+    return MiGA::Result.load(json) unless save
+
+    MiGA::Result.create(json, opts[:force]) do
+      r = send("add_result_#{task}", base, opts) if File.exist?("#{base}.done")
+      unless r.nil?
+        r.save
+        pull_hook(:on_result_ready, r.key)
+      end
+    end
+  end
+
+  ##
+  # Get the result MiGA::Result in this object identified by the symbol +task+
+  def result(task)
+    task = task.to_sym
+    return nil if result_dirs[task].nil?
+    MiGA::Result.load(
+      "#{project.path}/data/#{result_dirs[task]}/#{result_base}.json"
+    )
+  end
+
+  ##
+  # Get all the results (Array of MiGA::Result) in this object
+  def results
+    result_dirs.keys.map { |k| result k }.compact
+  end
+
+  ##
+  # For each result execute the 2-ary block: key symbol and MiGA::Result
+  def each_result
+    results.each { |res| yield(res.key, res) }
+  end
+
+  ##
+  # Get a result as MiGA::Result for the object with key +task+.
+  # This is equivalent to +add_result(task, false)+.
+  def get_result(task)
+    add_result(task, false)
+  end
+end

data/lib/miga/daemon.rb

@@ -198,7 +198,7 @@ class MiGA::Daemon < MiGA::MiGA
     @jobs_running.select! do |job|
       ongoing = case job[:job].to_s
       when 'd'
-        !job[:ds].next_preprocessing(false).nil?
+        !job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
       when 'p'
         !project.next_task(nil, false).nil?
       else
@@ -244,6 +244,8 @@ class MiGA::Daemon < MiGA::MiGA
       say 'MiGA:%s launched' % project.name
       say '-----------------------------------'
       load_status
+      say 'Configuration options:'
+      say @runopts.to_s
       @loop_i = 0
     end
     @loop_i += 1

data/lib/miga/dataset.rb

@@ -143,18 +143,6 @@ class MiGA::Dataset < MiGA::MiGA
     metadata[:inactive].nil? or !metadata[:inactive]
   end
 
-  ##
-  # Should I ignore +task+ for this dataset?
-  def ignore_task?(task)
-    return true unless is_active?
-    return !metadata["run_#{task}"] unless metadata["run_#{task}"].nil?
-    return true if task == :taxonomy and project.metadata[:ref_project].nil?
-    pattern = [true, false]
-    ( [@@_EXCLUDE_NOREF_TASKS_H[task], is_ref?     ] == pattern or
-      [@@_ONLY_MULTI_TASKS_H[task],    is_multi?   ] == pattern or
-      [@@_ONLY_NONMULTI_TASKS_H[task], is_nonmulti?] == pattern )
-  end
-
   ##
   # Returns an Array of +how_many+ duples (Arrays) sorted by AAI:
   # - +0+: A String with the name(s) of the reference dataset.
@@ -172,5 +160,5 @@ class MiGA::Dataset < MiGA::MiGA
       'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many])
   end
 
-end # class MiGA::Dataset
+end
 

data/lib/miga/dataset/hooks.rb

@@ -17,7 +17,6 @@ require 'miga/common/hooks'
 # - clear_run_counts()
 # - run_cmd(cmd)
 # Internal hooks:
-# - _pull_preprocessing_ready_hooks()
 # - _pull_result_hooks()
 module MiGA::Dataset::Hooks 
 
@@ -26,10 +25,7 @@ module MiGA::Dataset::Hooks
   def default_hooks
     {
       on_preprocessing_ready: [[:clear_run_counts]],
-      on_result_ready: [
-        [:_pull_result_hooks],
-        [:_pull_preprocessing_ready_hooks]
-      ]
+      on_result_ready: [[:_pull_result_hooks]]
     }
   end
 
@@ -57,17 +53,12 @@ module MiGA::Dataset::Hooks
     )
   end
 
-  ##
-  # Pull :dataset_ready hook if preprocessing is complete
-  def hook__pull_preprocessing_ready_hooks(_hook_args, _event_args)
-    pull_hook(:on_preprocessing_ready) if done_preprocessing?
-  end
-
   ##
   # Dataset Action :pull_result_hooks([], [res])
   # Pull the hook specific to the type of result
   def hook__pull_result_hooks(_hook_args, event_args)
     pull_hook(:"on_result_ready_#{event_args.first}", *event_args)
+    pull_hook(:on_preprocessing_ready) if done_preprocessing?
   end
 
 end

data/lib/miga/dataset/result.rb

@@ -1,68 +1,53 @@
-
 require 'sqlite3'
 require 'miga/result'
 require 'miga/dataset/base'
+require 'miga/common/with_result'
 
 ##
 # Helper module including specific functions to add dataset results
 module MiGA::Dataset::Result
   include MiGA::Dataset::Base
+  include MiGA::Common::WithResult
 
   ##
-  # Get the result MiGA::Result in this dataset identified by the symbol +k+
-  def result(k)
-    return nil if @@RESULT_DIRS[k.to_sym].nil?
-    MiGA::Result.load(
-      "#{project.path}/data/#{@@RESULT_DIRS[k.to_sym]}/#{name}.json"
-    )
+  # Return the basename for results
+  def result_base
+    name
   end
 
   ##
-  # Get all the results (Array of MiGA::Result) in this dataset
-  def results
-    @@RESULT_DIRS.keys.map { |k| result k }.compact
+  # Should I ignore +task+ for this dataset?
+  def ignore_task?(task)
+    why_ignore(task) != :execute
   end
 
   ##
-  # For each result executes the 2-ary block: key symbol and MiGA::Result
-  def each_result
-    @@RESULT_DIRS.each_key do |k|
-      yield(k, result(k)) unless result(k).nil?
-    end
-  end
-
-  ##
-  # Look for the result with symbol key +result_type+ and register it in the
-  # dataset. If +save+ is false, it doesn't register the result, but it still
-  # returns a result if the expected files are complete. The +opts+ hash
-  # controls result creation (if necessary). Supported values include:
-  # - +is_clean+: A Boolean indicating if the input files are clean
-  # - +force+: A Boolean indicating if the result must be re-indexed.
-  #   If true, it implies +save = true+
-  # Returns MiGA::Result or nil.
-  def add_result(result_type, save = true, opts = {})
-    dir = @@RESULT_DIRS[result_type]
-    return nil if dir.nil?
-    base = File.expand_path("data/#{dir}/#{name}", project.path)
-    if opts[:force]
-      FileUtils.rm("#{base}.json") if File.exist?("#{base}.json")
+  # Return a code explaining why a task is ignored.
+  # The values are symbols:
+  # - empty: the dataset has no data
+  # - inactive: the dataset is inactive
+  # - force: forced to ignore by metadata
+  # - project: incompatible project
+  # - noref: incompatible dataset, only for reference
+  # - multi: incompatible dataset, only for multi
+  # - nonmulti: incompatible dataset, only for nonmulti
+  # - execute: do not ignore, execute the task
+  def why_ignore(task)
+    if !is_active?
+      :inactive
+    elsif !metadata["run_#{task}"].nil?
+      metadata["run_#{task}"] ? :execute : :force
+    elsif task == :taxonomy && project.metadata[:ref_project].nil?
+      :project
+    elsif @@_EXCLUDE_NOREF_TASKS_H[task] && !is_ref?
+      :noref
+    elsif @@_ONLY_MULTI_TASKS_H[task] && !is_multi?
+      :multi
+    elsif @@_ONLY_NONMULTI_TASKS_H[task] && !is_nonmulti?
+      :nonmulti
     else
-      r_pre = MiGA::Result.load("#{base}.json")
-      return r_pre if (r_pre.nil? && !save) || !r_pre.nil?
+      :execute
     end
-    fun = "add_result_#{result_type}"
-    r = send(fun, base, opts) if File.exist?("#{base}.done")
-    return if r.nil?
-    r.save
-    pull_hook(:on_result_ready, result_type)
-    r
-  end
-
-  ##
-  # Gets a result as MiGA::Result for the datasets with +result_type+. This is
-  # equivalent to +add_result(result_type, false)+.
-  def get_result(result_type)
-    add_result(result_type, false)
   end
 
   ##
@@ -76,24 +61,22 @@ module MiGA::Dataset::Result
   end
 
   ##
-  # Returns the key symbol of the next task that needs to be executed. Passes
-  # +save+ to #add_result.
+  # Returns the key symbol of the next task that needs to be executed or nil.
+  # Passes +save+ to #add_result.
   def next_preprocessing(save = false)
-    after_first = false
-    first = first_preprocessing(save)
-    return nil if first.nil?
-    @@PREPROCESSING_TASKS.each do |t|
-      next if ignore_task? t
-      if after_first && add_result(t, save).nil?
+    first = first_preprocessing(save) or return nil
+    @@PREPROCESSING_TASKS[@@PREPROCESSING_TASKS.index(first) .. -1].find do |t|
+      if ignore_task? t
+        false
+      elsif add_result(t, save).nil?
         if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
           inactivate!
-          return nil
+          false
+        else
+          true
         end
-        return t
       end
-      after_first = (after_first || (t == first))
     end
-    nil
   end
 
   ##
@@ -133,14 +116,7 @@ module MiGA::Dataset::Result
   ##
   # Returns the status of +task+. The status values are symbols:
   # - -: the task is upstream from the initial input
-  # - ignore_empty: the dataset has no data
-  # - ignore_inactive: the dataset is inactive
-  # - ignore_force: forced to ignore by metadata
-  # - ignore_project: incompatible project
-  # - ignore_noref: incompatible dataset, only for reference
-  # - ignore_multi: incompatible dataset, only for multi
-  # - ignore_nonmulti: incompatible dataset, only for nonmulti
-  # - ignore: incompatible dataset, unknown reason
+  # - ignore_*: the task is to be ignored, see codes in #why_ignore
   # - complete: a task with registered results
   # - pending: a task queued to be performed
   def result_status(task)
@@ -152,21 +128,7 @@ module MiGA::Dataset::Result
           @@PREPROCESSING_TASKS.index(first_preprocessing)
       :-
     elsif ignore_task?(task)
-      if !is_active?
-        :ignore_inactive
-      elsif metadata["run_#{task}"]
-        :ignore_force
-      elsif task == :taxonomy && project.metadata[:ref_project].nil?
-        :ignore_project
-      elsif @@_EXCLUDE_NOREF_TASKS_H[task] && !is_ref?
-        :ignore_noref
-      elsif @@_ONLY_MULTI_TASKS_H[task] && !is_multi?
-        :ignore_multi
-      elsif @@_ONLY_NONMULTI_TASKS_H[task] && !is_nonmulti?
-        :ignore_nonmulti
-      else
-        :ignore
-      end
+      :"ignore_#{why_ignore task}"
     else
       :pending
     end

data/lib/miga/project/hooks.rb

@@ -16,7 +16,6 @@ require 'miga/common/hooks'
 # - run_lambda(lambda, args...)
 # - run_cmd(cmd)
 # Internal hooks:
-# - _pull_processing_ready_hooks()
 # - _pull_result_hooks()
 module MiGA::Project::Hooks
 
@@ -24,10 +23,7 @@ module MiGA::Project::Hooks
 
   def default_hooks
     {
-      on_result_ready: [
-        [:_pull_result_hooks],
-        [:_pull_processing_ready_hooks]
-      ]
+      on_result_ready: [[:_pull_result_hooks]]
     }
   end
 
@@ -44,17 +40,12 @@ module MiGA::Project::Hooks
     )
   end
 
-  ##
-  # Pull :dataset_ready hook if preprocessing is complete
-  def hook__pull_processing_ready_hooks(_hook_args, _event_args)
-    pull_hook(:on_processing_ready) if next_task(nil, false).nil?
-  end
-
   ##
   # Dataset Action :pull_result_hooks([], [res])
   # Pull the hook specific to the type of result
   def hook__pull_result_hooks(_hook_args, event_args)
     pull_hook(:"on_result_ready_#{event_args.first}", *event_args)
+    pull_hook(:on_processing_ready) if next_task(nil, false).nil?
   end
 
 end

data/lib/miga/project/result.rb

@@ -3,69 +3,47 @@
 
 require 'miga/result'
 require 'miga/project/base'
+require 'miga/common/with_result'
 
 ##
 # Helper module including specific functions to add project results.
 module MiGA::Project::Result
-
   include MiGA::Project::Base
+  include MiGA::Common::WithResult
 
   ##
-  # Get result identified by Symbol +name+, returns MiGA::Result.
-  def result(name)
-    dir = @@RESULT_DIRS[name.to_sym]
-    return nil if dir.nil?
-    MiGA::Result.load("#{path}/data/#{dir}/miga-project.json")
+  # Return the basename for results
+  def result_base
+    'miga-project'
   end
 
   ##
-  # Get all results, an Array of MiGA::Result.
-  def results
-    @@RESULT_DIRS.keys.map{ |k| result(k) }.reject{ |r| r.nil? }
-  end
-
-  ##
-  # For each result executes the 2-ary +blk+ block: key symbol and MiGA::Result.
-  def each_result(&blk)
-    @@RESULT_DIRS.keys.each do |k|
-      blk.call(k, result(k)) unless result(k).nil?
-    end
+  # Return itself, to simplify modules
+  def project
+    self
   end
 
   ##
-  # Add the result identified by Symbol +name+, and return MiGA::Result. Save
-  # the result if +save+. The +opts+ hash controls result creation (if
-  # necessary).
-  # Supported values include:
-  # - +force+: A Boolean indicating if the result must be re-indexed. If true,
-  #   it implies save=true.
-  def add_result(name, save = true, opts = {})
-    return nil if @@RESULT_DIRS[name].nil?
-    base = "#{path}/data/#{@@RESULT_DIRS[name]}/miga-project"
-    if opts[:force]
-      FileUtils.rm("#{base}.json") if File.exist?("#{base}.json")
-    else
-      r_pre = MiGA::Result.load("#{base}.json")
-      return r_pre if (r_pre.nil? && !save) || !r_pre.nil?
-    end
-    r = result_files_exist?(base, ".done") ?
-        send("add_result_#{name}", base) : nil
-    unless r.nil?
-      r.save
-      pull_hook(:on_result_ready, name)
-    end
-    r
+  # Is this +task+ to be bypassed?
+  def ignore_task?(task)
+    metadata["run_#{task}"] == false ||
+      (!is_clade? && @@INCLADE_TASKS.include?(task) &&
+        metadata["run_#{task}"] != true)
   end
 
   ##
   # Get the next distances task, saving intermediate results if +save+. Returns
   # a Symbol.
-  def next_distances(save = true) ; next_task(@@DISTANCE_TASKS, save) ; end
+  def next_distances(save = true)
+    next_task(@@DISTANCE_TASKS, save)
+  end
 
   ##
   # Get the next inclade task, saving intermediate results if +save+. Returns a
   # Symbol.
-  def next_inclade(save = true) ; next_task(@@INCLADE_TASKS, save) ; end
+  def next_inclade(save = true)
+    next_task(@@INCLADE_TASKS, save)
+  end
 
   ##
   # Get the next task from +tasks+, saving intermediate results if +save+.
@@ -74,101 +52,104 @@ module MiGA::Project::Result
   def next_task(tasks = nil, save = true)
     tasks ||= @@DISTANCE_TASKS + @@INCLADE_TASKS
     tasks.find do |t|
-      if metadata["run_#{t}"] == false or
-            (!is_clade? and @@INCLADE_TASKS.include?(t) and
-                  metadata["run_#{t}"] != true)
-        false
-      else
-        add_result(t, save).nil?
-      end
+      ignore_task?(t) ? false : add_result(t, save).nil?
     end
   end
 
-
   private
 
-    ##
-    # Internal alias for all add_result_*_distances.
-    def add_result_distances(base)
-      return nil unless result_files_exist?(base, %w[.Rdata .log .txt])
-      r = MiGA::Result.new("#{base}.json")
-      r.add_file(:rdata,  'miga-project.Rdata')
-      r.add_file(:matrix, 'miga-project.txt')
-      r.add_file(:log,    'miga-project.log')
-      r.add_file(:hist,   'miga-project.hist')
-      r
-    end
+  ##
+  # Add result of any type +:*_distances+ at +base+ (no +_opts+ supported).
+  def add_result_distances(base, _opts)
+    return nil unless result_files_exist?(base, %w[.Rdata .log .txt])
+    r = MiGA::Result.new("#{base}.json")
+    r.add_file(:rdata,  'miga-project.Rdata')
+    r.add_file(:matrix, 'miga-project.txt')
+    r.add_file(:log,    'miga-project.log')
+    r.add_file(:hist,   'miga-project.hist')
+    r
+  end
 
-    def add_result_clade_finding(base)
-      if result_files_exist?(base, %w[.empty])
-        r = MiGA::Result.new("#{base}.json")
-        r.add_file(:empty, 'miga-project.empty')
-        return r
-      end
-      return nil unless result_files_exist?(base, %w[.proposed-clades])
-      return nil unless is_clade? or result_files_exist?(base,
-          %w[.pdf .classif .medoids .class.tsv .class.nwk])
-      r = add_result_iter_clades(base)
-      r.add_file(:aai_tree,      'miga-project.aai.nwk')
-      r.add_file(:proposal,      'miga-project.proposed-clades')
-      r.add_file(:clades_aai90,  'miga-project.aai90-clades')
-      r.add_file(:clades_ani95,  'miga-project.ani95-clades')
-      r.add_file(:clades_gsp,  'miga-project.gsp-clades')
-      r.add_file(:medoids_gsp, 'miga-project.gsp-medoids')
-      r
+  ##
+  # Add result type +:clade_finding+ at +base+ (no +_opts+ supported).
+  def add_result_clade_finding(base, _opts)
+    if result_files_exist?(base, %w[.empty])
+      r = MiGA::Result.new("#{base}.json")
+      r.add_file(:empty, 'miga-project.empty')
+      return r
     end
-
-    def add_result_subclades(base)
-      if result_files_exist?(base, %w[.empty])
-        r = MiGA::Result.new("#{base}.json")
-        r.add_file(:empty, 'miga-project.empty')
-        return r
-      end
-      return nil unless result_files_exist?(base,
+    return nil unless result_files_exist?(base, %w[.proposed-clades])
+    return nil unless is_clade? or result_files_exist?(base,
         %w[.pdf .classif .medoids .class.tsv .class.nwk])
-      r = add_result_iter_clades(base)
-      r.add_file(:ani_tree, 'miga-project.ani.nwk')
-      r
-    end
+    r = add_result_iter_clades(base)
+    r.add_file(:aai_tree,      'miga-project.aai.nwk')
+    r.add_file(:proposal,      'miga-project.proposed-clades')
+    r.add_file(:clades_aai90,  'miga-project.aai90-clades')
+    r.add_file(:clades_ani95,  'miga-project.ani95-clades')
+    r.add_file(:clades_gsp,  'miga-project.gsp-clades')
+    r.add_file(:medoids_gsp, 'miga-project.gsp-medoids')
+    r
+  end
 
-    def add_result_iter_clades(base)
+  ##
+  # Add result type +:subclades+ at +base+ (no +_opts+ supported).
+  def add_result_subclades(base, _opts)
+    if result_files_exist?(base, %w[.empty])
       r = MiGA::Result.new("#{base}.json")
-      r.add_file(:report,      'miga-project.pdf')
-      r.add_file(:class_table, 'miga-project.class.tsv')
-      r.add_file(:class_tree,  'miga-project.class.nwk')
-      r.add_file(:classif,     'miga-project.classif')
-      r.add_file(:medoids,     'miga-project.medoids')
-      r
+      r.add_file(:empty, 'miga-project.empty')
+      return r
     end
+    return nil unless result_files_exist?(base,
+      %w[.pdf .classif .medoids .class.tsv .class.nwk])
+    r = add_result_iter_clades(base)
+    r.add_file(:ani_tree, 'miga-project.ani.nwk')
+    r
+  end
 
-    def add_result_ogs(base)
-      if result_files_exist?(base, %w[.empty])
-        r = MiGA::Result.new("#{base}.json")
-        r.add_file(:empty, 'miga-project.empty')
-        return r
-      end
-      return nil unless result_files_exist?(base, %w[.ogs .stats])
-      r = MiGA::Result.new("#{base}.json")
-      r.add_file(:ogs,   'miga-project.ogs')
-      r.add_file(:abc,   'miga-project.abc')
-      r.add_file(:stats, 'miga-project.stats')
-      r.add_file(:core_pan,      'miga-project.core-pan.tsv')
-      r.add_file(:core_pan_plot, 'miga-project.core-pan.pdf')
-      r
-    end
+  ##
+  # Helper function for clade iterations.
+  def add_result_iter_clades(base)
+    r = MiGA::Result.new("#{base}.json")
+    r.add_file(:report,      'miga-project.pdf')
+    r.add_file(:class_table, 'miga-project.class.tsv')
+    r.add_file(:class_tree,  'miga-project.class.nwk')
+    r.add_file(:classif,     'miga-project.classif')
+    r.add_file(:medoids,     'miga-project.medoids')
+    r
+  end
 
-    def add_result_project_stats(base)
-      return nil unless
-        result_files_exist?(base, %w[.taxonomy.json .metadata.db])
+  ##
+  # Add result type +:ogs+ at +base+ (no +_opts+ supported).
+  def add_result_ogs(base, _opts)
+    if result_files_exist?(base, %w[.empty])
       r = MiGA::Result.new("#{base}.json")
-      r.add_file(:taxonomy_index, 'miga-project.taxonomy.json')
-      r.add_file(:metadata_index, 'miga-project.metadata.db')
-      r
+      r.add_file(:empty, 'miga-project.empty')
+      return r
     end
+    return nil unless result_files_exist?(base, %w[.ogs .stats])
+    r = MiGA::Result.new("#{base}.json")
+    r.add_file(:ogs,   'miga-project.ogs')
+    r.add_file(:abc,   'miga-project.abc')
+    r.add_file(:stats, 'miga-project.stats')
+    r.add_file(:core_pan,      'miga-project.core-pan.tsv')
+    r.add_file(:core_pan_plot, 'miga-project.core-pan.pdf')
+    r
+  end
+
+  ##
+  # Add result type +:project_stats+ at +base+ (no +_opts+ supported).
+  def add_result_project_stats(base, _opts)
+    return nil unless
+      result_files_exist?(base, %w[.taxonomy.json .metadata.db])
+    r = MiGA::Result.new("#{base}.json")
+    r.add_file(:taxonomy_index, 'miga-project.taxonomy.json')
+    r.add_file(:metadata_index, 'miga-project.metadata.db')
+    r
+  end
 
-    alias add_result_haai_distances add_result_distances
-    alias add_result_aai_distances add_result_distances
-    alias add_result_ani_distances add_result_distances
-    alias add_result_ssu_distances add_result_distances
+  alias add_result_haai_distances add_result_distances
+  alias add_result_aai_distances add_result_distances
+  alias add_result_ani_distances add_result_distances
+  alias add_result_ssu_distances add_result_distances
 
 end

data/lib/miga/remote_dataset.rb

@@ -1,6 +1,7 @@
 # @package MiGA
 # @license Artistic-2.0
 
+require 'miga/taxonomy'
 require 'miga/remote_dataset/download'
 
 ##
@@ -203,7 +204,7 @@ class MiGA::RemoteDataset < MiGA::MiGA
       when ''
         metadata[:is_type] = false
         metadata[:is_ref_type] = false
-      when 'assembly from reference material'
+      when 'assembly from reference material', 'assembly designated as reftype'
         metadata[:is_type] = false
         metadata[:is_ref_type] = true
         metadata[:type_rel] = from_type

data/lib/miga/result.rb

@@ -14,19 +14,28 @@ class MiGA::Result < MiGA::MiGA
   include MiGA::Result::Stats
 
   # Class-level
+  class << self
+    ##
+    # Check if the result described by the JSON in +path+ already exists
+    def exist?(path)
+      File.exist? path
+    end
 
-  ##
-  # Check if the result described by the JSON in +path+ already exists.
-  def self.exist?(path)
-    File.exist? path
-  end
+    ##
+    # Load the result described by the JSON in +path+.
+    # Returns MiGA::Result if it already exists, nil otherwise.
+    def load(path)
+      return nil unless MiGA::Result.exist? path
+      MiGA::Result.new(path)
+    end
 
-  ##
-  # Load the result described by the JSON in +path+. Returns MiGA::Result if it
-  # already exists, nil otherwise.
-  def self.load(path)
-    return nil unless MiGA::Result.exist? path
-    MiGA::Result.new(path)
+    def create(path, force = false, &blk)
+      FileUtils.rm(path) if force && File.exist?(path)
+      r_pre = self.load(path)
+      return r_pre unless r_pre.nil?
+      yield
+      self.load(path)
+    end
   end
 
   # Instance-level

data/lib/miga/result/source.rb

@@ -18,9 +18,13 @@ module MiGA::Result::Source
   ##
   # Detect the result key assigned to this result
   def key
-    @key ||= MiGA::Result.RESULT_DIRS.find do |k, v|
-      "data/#{v}" == relative_dir
-    end.first
+    @key ||= if relative_dir == 'data/90.stats' && file_path(:metadata_index)
+      :project_stats
+    else
+      MiGA::Result.RESULT_DIRS.find do |k, v|
+        "data/#{v}" == relative_dir
+      end.first
+    end
   end
 
   ##

data/lib/miga/result/stats.rb

@@ -1,4 +1,5 @@
 
+require 'zlib'
 require 'miga/result/base'
 
 ##
@@ -75,6 +76,16 @@ module MiGA::Result::Stats
       stats[:coding_density] =
         [300.0 * s[:tot] / asm[:stats][:total_length][0], '%']
     end
+    if file_path(:gff3) && file_path(:gff3) =~ /\.gz/
+      Zlib::GzipReader.open(file_path(:gff3)) do |fh|
+        fh.each do |ln|
+          if ln =~ /^# Model Data:.*;transl_table=(\d+);/
+            stats[:codon_table] = $1
+            break
+          end
+        end
+      end
+    end
     stats
   end
 

data/lib/miga/tax_dist.rb

@@ -31,6 +31,7 @@ module MiGA::TaxDist
     # correspond to the p-values of +test+ (one of +:intax+ or +:novel+)
     # with options +opts+. See +aai_path+ for supported options.
     def aai_pvalues(aai, test, opts = {})
+      y = {}
       Zlib::GzipReader.open(aai_path(test, opts)) do |fh|
         keys = nil
         fh.each_line do |ln|
@@ -46,11 +47,13 @@ module MiGA::TaxDist
               rank = i.zero? ? :root : MiGA::Taxonomy.KNOWN_RANKS[i]
               vals[rank] = v.to_f
             end
-            return vals
+            y = vals
+            break
           end
         end
+        fh.rewind # to avoid warnings caused by the break above
       end
-      {}
+      y
     end
 
     ##

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.6, 3, 0]
+  VERSION = [0.6, 3, 1]
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 3, 27)
+  VERSION_DATE = Date.new(2020, 4, 11)
 
   ##
   # Reference of MiGA.

data/scripts/cds.bash

@@ -36,7 +36,7 @@ case "$TYPE" in
         | perl -pe 's/[^A-Z]//ig' | wc -c | awk '{print $1}')
       echo "# codon table $ct total length: $C_LEN aa" \
         >> "${DATASET}.ct.t"
-      if [[ $C_LEN > $(($P_LEN * 11 / 10)) ]] ; then
+      if [[ $C_LEN -gt $(($P_LEN * 11 / 10)) ]] ; then
         for x in faa fna gff3 ; do
           mv "${DATASET}.$x.$ct" "${DATASET}.$x"
         done

data/test/daemon_test.rb

@@ -115,7 +115,7 @@ class DaemonTest < Test::Unit::TestCase
       assert(l[0] =~ /-{20}\n/)
       assert(l[1] =~ /MiGA:#{p.name} launched/)
       assert(l[2] =~ /-{20}\n/)
-      assert(l[3] =~ /Probing running jobs\n/)
+      assert(l[5] =~ /Probing running jobs\n/)
     end
   ensure
     begin

data/test/project_test.rb

@@ -2,7 +2,6 @@ require 'test_helper'
 require 'miga/project'
 
 class ProjectTest < Test::Unit::TestCase
-
   def setup
     $tmp = Dir.mktmpdir
     ENV['MIGA_HOME'] = $tmp
@@ -16,6 +15,17 @@ class ProjectTest < Test::Unit::TestCase
     ENV['MIGA_HOME'] = nil
   end
 
+  def create_result_files(project, res, exts)
+    d = MiGA::Project.RESULT_DIRS[res]
+    (['.done'] + exts).each do |x|
+      assert_nil(
+        project.add_result(res),
+        "Premature registration of result #{res} at extension #{x}"
+      )
+      FileUtils.touch(File.join(project.path, "data/#{d}/miga-project#{x}"))
+    end
+  end
+
   def test_class_load
     assert_nil(MiGA::Project.load($tmp + '/O_o'))
     assert_equal(MiGA::Project, MiGA::Project.load($p1.path).class)
@@ -47,7 +57,8 @@ class ProjectTest < Test::Unit::TestCase
     assert_equal(MiGA::Dataset, d.class)
     assert_equal([d], p.datasets)
     assert_equal(['d1'], p.dataset_names)
-    p.each_dataset{ |ds| assert_equal(d, ds) }
+    p.each_dataset { |ds| assert_equal(d, ds) }
+    p.each_dataset { |n, ds| assert_equal(n, ds.name) }
     dr = p.unlink_dataset('d1')
     assert_equal(d, dr)
     assert_equal([], p.datasets)
@@ -71,12 +82,10 @@ class ProjectTest < Test::Unit::TestCase
     assert_equal(1, p2.datasets.size)
     assert_equal(MiGA::Dataset, p2.dataset('d1').class)
     assert_equal(1, p2.dataset('d1').results.size)
-    assert(File.exist?(
-      File.expand_path("data/01.raw_reads/#{d1.name}.1.fastq", p2.path)
-    ))
-    assert(File.exist?(
-      File.expand_path("metadata/#{d1.name}.json", p2.path)
-    ))
+    assert(
+      File.exist?(File.join(p2.path, "data/01.raw_reads/#{d1.name}.1.fastq"))
+    )
+    assert(File.exist?(File.join(p2.path, "metadata/#{d1.name}.json")))
   end
 
   def test_add_result
@@ -85,7 +94,7 @@ class ProjectTest < Test::Unit::TestCase
     %w[.Rdata .log .txt .done].each do |x|
       assert_nil(p1.add_result(:haai_distances))
       FileUtils.touch(
-        File.expand_path("data/09.distances/01.haai/miga-project#{x}",p1.path)
+        File.join(p1.path, "data/09.distances/01.haai/miga-project#{x}")
       )
     end
     assert_equal(MiGA::Result, p1.add_result(:haai_distances).class)
@@ -93,12 +102,12 @@ class ProjectTest < Test::Unit::TestCase
 
   def test_result
     p1 = $p1
-    assert_nil(p1.result :n00b)
-    assert_nil(p1.result :project_stats)
+    assert_nil(p1.result(:n00b))
+    assert_nil(p1.result(:project_stats))
     File.open(
       File.expand_path('data/90.stats/miga-project.json', p1.path), 'w'
     ) { |fh| fh.puts '{}' }
-    assert_not_nil(p1.result :project_stats)
+    assert_not_nil(p1.result(:project_stats))
     assert_equal(1, p1.results.size)
   end
 
@@ -109,64 +118,48 @@ class ProjectTest < Test::Unit::TestCase
     assert(!p1.done_preprocessing?)
     FileUtils.touch(File.expand_path("data/90.stats/#{d1.name}.done", p1.path))
     assert(p1.done_preprocessing?)
+    assert_nil(p1.next_inclade)
+    p1.metadata[:type] = :clade
+    assert_equal(:subclades, p1.next_inclade)
 
-    # Project stats
-    assert_equal(:project_stats, p1.next_distances)
-    d = MiGA::Project.RESULT_DIRS[:project_stats]
-    %w[.done .taxonomy.json .metadata.db].each do |x|
-      assert_nil(
-        p1.add_result(:project_stats),
-        "Premature registration of result project_stats at extension #{x}."
-      )
-      FileUtils.touch(File.expand_path("data/#{d}/miga-project#{x}", p1.path))
-    end
-    assert_equal(MiGA::Result, p1.add_result(:project_stats).class,
-      'Imposible to add project_stats result.')
-
-    # Distances
-    [:haai_distances, :aai_distances, :ani_distances].each do |r|
-      assert_equal(r, p1.next_distances)
-      assert_equal(Symbol, p1.next_distances.class)
-      d = MiGA::Project.RESULT_DIRS[r]
-      %w[.done .Rdata .log .txt].each do |x|
-        assert_nil(
-          p1.add_result(r),
-          "Premature registration of result #{r} at extension #{x}."
-        )
-        FileUtils.touch(File.expand_path("data/#{d}/miga-project#{x}", p1.path))
-      end
-      assert_equal(MiGA::Result, p1.add_result(r).class,
-        "Imposible to add #{r} result.")
+    # Project tasks
+    expected_files = {
+      project_stats: %w[.taxonomy.json .metadata.db],
+      haai_distances: %w[.Rdata .log .txt],
+      aai_distances: %w[.Rdata .log .txt],
+      ani_distances: %w[.Rdata .log .txt],
+      clade_finding: %w[.pdf .classif .medoids
+        .class.tsv .class.nwk .proposed-clades],
+      subclades: %w[.pdf .classif .medoids .class.tsv .class.nwk],
+      ogs: %w[.ogs .stats]
+    }
+    expected_files.each do |r, exts|
+      assert_equal(r, p1.next_task)
+      create_result_files(p1, r, exts)
+      assert_not_nil(p1.add_result(r), "Imposible to add #{r} result")
     end
-    assert_equal(:clade_finding, p1.next_distances)
+    assert_nil(p1.next_task)
+    p1.each_result { |k, r| assert_equal(k, r.key) }
+  end
 
-    # Clades
-    assert_nil(p1.next_inclade)
+  def test_empty_results
+    p1 = $p1
     p1.metadata[:type] = :clade
-    res = [
-      [:clade_finding,
-        %w[.pdf .classif .medoids .class.tsv .class.nwk .proposed-clades]],
-      [:subclades, %w[.pdf .classif .medoids .class.tsv .class.nwk]],
-      [:ogs, %w[.ogs .stats]]
-    ]
-    res.each do |rr|
-      (r, xs) = rr
-      d = MiGA::Project.RESULT_DIRS[r]
-      assert_equal(Symbol, p1.next_inclade.class)
-      ([".done"] + xs).each do |x|
-        assert_nil(
-          p1.add_result(r),
-          "Premature registration of result #{r} at extension #{x}."
-        )
-        FileUtils.touch(File.expand_path("data/#{d}/miga-project#{x}", p1.path))
-      end
-      assert_equal(
-        MiGA::Result,
-        p1.add_result(r).class,
-        "Impossible to add #{r} result."
-      )
+    %i[clade_finding subclades ogs].each do |r|
+      assert_nil(p1.add_result(r), "Unexpected result exists: #{r}")
+      create_result_files(p1, r, %w[.empty])
+      assert_not_nil(p1.add_result(r), "Cannot register emtpy task: #{r}")
     end
-    assert_nil(p1.next_inclade)
   end
 
+  def test_force_result
+    p1 = $p1
+    create_result_files(p1, :ogs, %w[.empty])
+    date1 = p1.add_result(:ogs)[:created]
+    sleep(1)
+    date2 = p1.add_result(:ogs, true, force: false)[:created]
+    assert_equal(date1, date2)
+    date3 = p1.add_result(:ogs, true, force: true)[:created]
+    assert_not_equal(date1, date3)
+  end
 end

data/test/remote_dataset_test.rb

@@ -30,10 +30,10 @@ class RemoteDatasetTest < Test::Unit::TestCase
 
   def test_rest
     hiv2 = 'M30502.1'
-    {embl: :ebi, nuccore: :ncbi}.each do |db, universe|
+    { embl: :ebi, nuccore: :ncbi }.each do |db, universe|
       rd = MiGA::RemoteDataset.new(hiv2, db, universe)
       assert_equal([hiv2], rd.ids)
-      omit_if(!$remote_tests, 'Remote access is error-prone.')
+      omit_if(!$remote_tests, 'Remote access is error-prone')
       tx = rd.get_ncbi_taxonomy
       msg = "Failed on #{universe}:#{db}"
       assert_equal(MiGA::Taxonomy, tx.class, msg)
@@ -57,7 +57,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
     n = 'Cjac_L14'
     rd = MiGA::RemoteDataset.new(cjac, :assembly_gz, :web)
     assert_equal([cjac], rd.ids)
-    omit_if(!$remote_tests, 'Remote access is error-prone.')
+    omit_if(!$remote_tests, 'Remote access is error-prone')
     p = $p1
     assert_nil(p.dataset(n))
     rd.save_to(p, n)
@@ -66,9 +66,54 @@ class RemoteDatasetTest < Test::Unit::TestCase
     assert_equal(MiGA::Result, p.dataset(n).result(:assembly).class)
   end
 
+  def test_asm_acc2id
+    omit_if(!$remote_tests, 'Remote access is error-prone')
+    assert_nil(MiGA::RemoteDataset.ncbi_asm_acc2id('NotAnAccession'))
+    id = MiGA::RemoteDataset.ncbi_asm_acc2id('GCA_004684205.1')
+    assert_equal('2514661', id)
+    assert_equal(id, MiGA::RemoteDataset.ncbi_asm_acc2id(id))
+  end
+
+  def test_update_metadata
+    omit_if(!$remote_tests, 'Remote access is error-prone')
+    hiv1 = 'GCF_000856385.1'
+    d1 = MiGA::Dataset.new($p1, 'd1')
+    assert_nil(d1.metadata[:ncbi_assembly])
+    rd = MiGA::RemoteDataset.new(hiv1, :assembly, :ncbi)
+    rd.update_metadata(d1, passthrough: 123, metadata_only: true)
+    assert_equal(123, d1.metadata[:passthrough])
+    assert_equal(hiv1, d1.metadata[:ncbi_assembly])
+    assert_equal('Lentivirus', d1.metadata[:tax][:g])
+  end
+
+  def test_type_status_asm
+    omit_if(!$remote_tests, 'Remote access is error-prone')
+    rd = MiGA::RemoteDataset.new('GCF_000018105.1', :assembly, :ncbi)
+    assert(rd.get_metadata[:is_type])
+  end
+
+  def test_nontype_status_asm
+    omit_if(!$remote_tests, 'Remote access is error-prone')
+    rd = MiGA::RemoteDataset.new('GCA_004684205.1', :assembly, :ncbi)
+    assert(!rd.get_metadata[:is_type])
+  end
+
+  def test_type_status_nuccore
+    omit_if(!$remote_tests, 'Remote access is error-prone')
+    rd = MiGA::RemoteDataset.new('NC_019748.1', :nuccore, :ncbi)
+    assert(rd.get_metadata[:is_type])
+  end
+
+  def test_ref_type_status
+    omit_if(!$remote_tests, 'Remote access is error-prone')
+    rd = MiGA::RemoteDataset.new('GCA_002849345', :assembly, :ncbi)
+    assert(!rd.get_metadata[:is_type])
+    assert(rd.get_metadata[:is_ref_type])
+  end
+
   # This test is too expensive (too much time to run it!)
   #def test_net_timeout
-  #  omit_if(!$remote_tests, "Remote access is error-prone.")
+  #  omit_if(!$remote_tests, "Remote access is error-prone")
   #  bad = "ftp://example.com/miga"
   #  rd = MiGA::RemoteDataset.new(bad, :assembly, :web)
   #  assert_raise(Net::ReadTimeout) { rd.save_to($p1, "bad") }

data/test/tax_dist_test.rb

@@ -26,7 +26,6 @@ class TaxDistTest < Test::Unit::TestCase
     assert_lt(close_intax[:s], 0.05)
 
     distant_novel = MiGA::TaxDist.aai_pvalues(35.0, :novel, engine: :diamond)
-    $stderr.puts distant_novel
     assert_gt(distant_novel[:root], 0.05)
     assert_lt(distant_novel[:g], 0.05)
     assert_nil(distant_novel[:ns])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.6.3.0
+  version: 0.6.3.1
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-27 00:00:00.000000000 Z
+date: 2020-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -157,6 +157,7 @@ files:
 - lib/miga/common/format.rb
 - lib/miga/common/hooks.rb
 - lib/miga/common/path.rb
+- lib/miga/common/with_result.rb
 - lib/miga/daemon.rb
 - lib/miga/daemon/base.rb
 - lib/miga/dataset.rb