miga-base 0.5.10.0 → 0.6.0.0

data/lib/miga/dataset/result.rb

@@ -6,21 +6,22 @@ require 'miga/dataset/base'
 ##
 # Helper module including specific functions to add dataset results.
 module MiGA::Dataset::Result
-  
+
   include MiGA::Dataset::Base
-  
+
   ##
   # Get the result MiGA::Result in this dataset identified by the symbol +k+.
   def result(k)
     return nil if @@RESULT_DIRS[k.to_sym].nil?
     MiGA::Result.load(
-      "#{project.path}/data/#{@@RESULT_DIRS[k.to_sym]}/#{name}.json" )
+      "#{project.path}/data/#{@@RESULT_DIRS[k.to_sym]}/#{name}.json"
+    )
   end
-  
+
   ##
   # Get all the results (Array of MiGA::Result) in this dataset.
   def results ; @@RESULT_DIRS.keys.map{ |k| result k }.compact ; end
-  
+
   ##
   # For each result executes the 2-ary +blk+ block: key symbol and MiGA::Result.
   def each_result(&blk)
@@ -28,15 +29,15 @@ module MiGA::Dataset::Result
       blk.call(k, result(k)) unless result(k).nil?
     end
   end
-  
+
   ##
   # Look for the result with symbol key +result_type+ and register it in the
   # dataset. If +save+ is false, it doesn't register the result, but it still
   # returns a result if the expected files are complete. The +opts+ hash
   # controls result creation (if necessary). Supported values include:
-  # - +is_clean+: A Boolean indicating if the input files are clean.
-  # - +force+: A Boolean indicating if the result must be re-indexed. If true,
-  #   it implies save=true.
+  # - +is_clean+: A Boolean indicating if the input files are clean
+  # - +force+: A Boolean indicating if the result must be re-indexed.
+  #   If true, it implies +save = true+
   # Returns MiGA::Result or nil.
   def add_result(result_type, save = true, opts = {})
     dir = @@RESULT_DIRS[result_type]
@@ -46,11 +47,14 @@ module MiGA::Dataset::Result
       FileUtils.rm("#{base}.json") if File.exist?("#{base}.json")
     else
       r_pre = MiGA::Result.load("#{base}.json")
-      return r_pre if (r_pre.nil? and not save) or not r_pre.nil?
+      return r_pre if (r_pre.nil? && !save) || !r_pre.nil?
     end
     r = File.exist?("#{base}.done") ?
         self.send("add_result_#{result_type}", base, opts) : nil
-    r.save unless r.nil?
+    unless r.nil?
+      r.save
+      pull_hook(:on_result_ready, result_type)
+    end
     r
   end
 
@@ -68,7 +72,7 @@ module MiGA::Dataset::Result
       not ignore_task?(t) and not add_result(t, save).nil?
     end
   end
-  
+
   ##
   # Returns the key symbol of the next task that needs to be executed. Passes
   # +save+ to #add_result.
@@ -95,7 +99,7 @@ module MiGA::Dataset::Result
   def done_preprocessing?(save = false)
     !first_preprocessing(save).nil? and next_preprocessing(save).nil?
   end
-  
+
   ##
   # Returns an array indicating the stage of each task (sorted by execution
   # order). The values are integers:
@@ -300,14 +304,17 @@ module MiGA::Dataset::Result
     # Add result type +:mytaxa+ at +base+ (no +_opts+ supported).
     def add_result_mytaxa(base, _opts)
       if is_multi?
-        return nil unless result_files_exist?(base, ".mytaxa") or
-          result_files_exist?(base, ".nomytaxa.txt")
+        return nil unless result_files_exist?(base, '.mytaxa') or
+          result_files_exist?(base, '.nomytaxa.txt')
         r = MiGA::Result.new("#{base}.json")
-        add_files_to_ds_result(r, name, mytaxa: ".mytaxa", blast: ".blast",
-          mytaxain: ".mytaxain", nomytaxa: ".nomytaxa.txt",
-          species: ".mytaxa.Species.txt", genus: ".mytaxa.Genus.txt",
-          phylum: ".mytaxa.Phylum.txt", innominate: ".mytaxa.innominate",
-          kronain: ".mytaxa.krona", krona: ".html")
+        add_files_to_ds_result(
+          r, name,
+          mytaxa: '.mytaxa', blast: '.blast',
+          mytaxain: '.mytaxain', nomytaxa: '.nomytaxa.txt',
+          species: '.mytaxa.Species.txt', genus: '.mytaxa.Genus.txt',
+          phylum: '.mytaxa.Phylum.txt', innominate: '.mytaxa.innominate',
+          kronain: '.mytaxa.krona', krona: '.html'
+        )
       else
         MiGA::Result.new("#{base}.json")
       end
@@ -357,7 +364,7 @@ module MiGA::Dataset::Result
     def add_result_stats(base, _opts)
       MiGA::Result.new("#{base}.json")
     end
-    
+
     ##
     # Add result type +:distances+ for _multi_ datasets at +base+.
     def add_result_distances_multi(base)
@@ -383,10 +390,13 @@ module MiGA::Dataset::Result
         result_files_exist?(base, %w[.aai-medoids.tsv .aai.db]) or
         result_files_exist?(base, %w[.ani-medoids.tsv .ani.db])
       r = MiGA::Result.new("#{base}.json")
-      add_files_to_ds_result(r, name, aai_medoids: ".aai-medoids.tsv",
-        haai_db: ".haai.db", aai_db: ".aai.db", ani_medoids: ".ani-medoids.tsv",
-        ani_db: ".ani.db", ref_tree: ".nwk", ref_tree_pdf: ".nwk.pdf",
-        intax_test: ".intax.txt")
+      add_files_to_ds_result(
+        r, name,
+        aai_medoids: '.aai-medoids.tsv',
+        haai_db: '.haai.db', aai_db: '.aai.db', ani_medoids: '.ani-medoids.tsv',
+        ani_db: '.ani.db', ref_tree: '.nwk', ref_tree_pdf: '.nwk.pdf',
+        intax_test: '.intax.txt'
+      )
     end
 
     ##

data/lib/miga/project.rb

@@ -4,7 +4,7 @@
 require 'miga/dataset'
 require 'miga/project/result'
 require 'miga/project/dataset'
-require 'miga/project/plugins'
+require 'miga/project/hooks'
 
 ##
 # MiGA representation of a project.
@@ -12,7 +12,7 @@ class MiGA::Project < MiGA::MiGA
   
   include MiGA::Project::Result
   include MiGA::Project::Dataset
-  include MiGA::Project::Plugins
+  include MiGA::Project::Hooks
 
   ##
   # Absolute path to the project folder.
@@ -35,7 +35,6 @@ class MiGA::Project < MiGA::MiGA
     @path = File.absolute_path(path)
     self.create if not update and not Project.exist? self.path
     self.load if self.metadata.nil?
-    self.load_plugins
     self.metadata[:type] = :mixed if type.nil?
     raise "Unrecognized project type: #{type}." if @@KNOWN_TYPES[type].nil?
   end
@@ -54,6 +53,7 @@ class MiGA::Project < MiGA::MiGA
       {datasets: [], name: File.basename(path)})
     d_path = File.expand_path('daemon/daemon.json', path)
     File.open(d_path, 'w') { |fh| fh.puts '{}' } unless File.exist? d_path
+    pull_hook :on_create
     self.load
   end
 
@@ -67,6 +67,7 @@ class MiGA::Project < MiGA::MiGA
   # Save any changes persistently, regardless of +do_not_save+.
   def save!
     metadata.save
+    pull_hook :on_save
     self.load
   end
 
@@ -77,6 +78,7 @@ class MiGA::Project < MiGA::MiGA
     @dataset_names_hash = nil
     @metadata = MiGA::Metadata.load "#{path}/miga.project.json"
     raise "Couldn't find project metadata at #{path}" if metadata.nil?
+    pull_hook :on_load
   end
 
   ##

data/lib/miga/project/dataset.rb

@@ -56,6 +56,7 @@ module MiGA::Project::Dataset
       @metadata[:datasets] << name
       @dataset_names_hash = nil # Ensure loading even if +do_not_save+ is true
       save
+      pull_hook(:on_add_dataset, name)
     end
     dataset(name)
   end
@@ -67,6 +68,7 @@ module MiGA::Project::Dataset
     return nil if d.nil?
     self.metadata[:datasets].delete(name)
     save
+    pull_hook(:on_unlink_dataset, name)
     d
   end
 
@@ -121,7 +123,7 @@ module MiGA::Project::Dataset
   ##
   # Are all the datasets in the project preprocessed? Save intermediate results
   # if +save+ (until the first incomplete dataset is reached).
-  def done_preprocessing?(save=true)
+  def done_preprocessing?(save = true)
     dataset_names.each do |dn|
       ds = dataset(dn)
       return false if ds.is_ref? and not ds.done_preprocessing?(save)

data/lib/miga/project/hooks.rb

@@ -0,0 +1,60 @@
+
+require 'miga/common/hooks'
+
+##
+# Helper module including specific functions to handle project hooks.
+# Supported events:
+# - on_create(): When created
+# - on_load(): When loaded
+# - on_save(): When saved
+# - on_add_dataset(dataset): When a dataset is added, with name +dataset+
+# - on_unlink_dataset(dataset): When dataset with name +dataset+ is unlinked
+# - on_result_ready(result): When any result is ready, with key +result+
+# - on_result_ready_{result}(): When +result+ is ready
+# - on_processing_ready(): When preprocessing is complete
+# Supported hooks:
+# - run_lambda(lambda, args...)
+# - run_cmd(cmd)
+# Internal hooks:
+# - _pull_processing_ready_hooks()
+# - _pull_result_hooks()
+module MiGA::Project::Hooks
+
+  include MiGA::Common::Hooks
+
+  def default_hooks
+    {
+      on_result_ready: [
+        [:_pull_result_hooks],
+        [:_pull_processing_ready_hooks]
+      ]
+    }
+  end
+
+  ##
+  # Run +cmd+ in the command-line with {{variables}}: project, miga,
+  # object (as defined by the event, if any)
+  # - +hook_args+: +[cmd]+
+  # - +event_args+: +[object (optional)]+
+  def hook_run_cmd(hook_args, event_args)
+    Process.wait(
+      spawn hook_args.first.miga_variables(
+        project: path, miga: MiGA::MiGA.root_path, object: event_args.first
+      )
+    )
+  end
+
+  ##
+  # Pull :dataset_ready hook if preprocessing is complete
+  def hook__pull_processing_ready_hooks(_hook_args, _event_args)
+    pull_hook(:on_processing_ready) if next_task(nil, false).nil?
+  end
+
+  ##
+  # Dataset Action :pull_result_hooks([], [res])
+  # Pull the hook specific to the type of result
+  def hook__pull_result_hooks(_hook_args, event_args)
+    pull_hook(:"on_result_ready_#{event_args.first}", *event_args)
+  end
+
+end

data/lib/miga/project/result.rb

@@ -46,11 +46,14 @@ module MiGA::Project::Result
       FileUtils.rm("#{base}.json") if File.exist?("#{base}.json")
     else
       r_pre = MiGA::Result.load("#{base}.json")
-      return r_pre if (r_pre.nil? and not save) or not r_pre.nil?
+      return r_pre if (r_pre.nil? && !save) || !r_pre.nil?
     end
     r = result_files_exist?(base, ".done") ?
         send("add_result_#{name}", base) : nil
-    r.save unless r.nil?
+    unless r.nil?
+      r.save
+      pull_hook(:on_result_ready, name)
+    end
     r
   end
 
@@ -69,7 +72,7 @@ module MiGA::Project::Result
   # If +tasks+ is +nil+ (default), it uses the entire list of tasks.
   # Returns a Symbol.
   def next_task(tasks = nil, save = true)
-    tasks ||= @@DISTANCE_TASKS+@@INCLADE_TASKS
+    tasks ||= @@DISTANCE_TASKS + @@INCLADE_TASKS
     tasks.find do |t|
       if metadata["run_#{t}"] == false or
             (!is_clade? and @@INCLADE_TASKS.include?(t) and

data/lib/miga/result.rb

@@ -1,19 +1,25 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require "miga/result/dates"
+require 'miga/result/dates'
+require 'miga/result/source'
+require 'miga/result/stats'
 
 ##
 # The result from a task run. It can be project-wide or dataset-specific.
 class MiGA::Result < MiGA::MiGA
-  
+
   include MiGA::Result::Dates
-  
+  include MiGA::Result::Source
+  include MiGA::Result::Stats
+
   # Class-level
-  
+
   ##
   # Check if the result described by the JSON in +path+ already exists.
-  def self.exist?(path) File.exist? path end
+  def self.exist?(path)
+    File.exist? path
+  end
 
   ##
   # Load the result described by the JSON in +path+. Returns MiGA::Result if it
@@ -32,14 +38,14 @@ class MiGA::Result < MiGA::MiGA
   ##
   # Array of MiGA::Result objects nested within the result (if any).
   attr_reader :results
-  
+
   ##
   # Load or create the MiGA::Result described by the JSON file +path+.
   def initialize(path)
-    @path = path
-    MiGA::Result.exist?(path) ? self.load : create
+    @path = File.absolute_path(path)
+    MiGA::Result.exist?(@path) ? self.load : create
   end
-  
+
   ##
   # Is the result clean? Returns Boolean.
   def clean? ; !! self[:clean] ; end
@@ -109,7 +115,7 @@ class MiGA::Result < MiGA::MiGA
   ##
   # Initialize and #save empty result.
   def create
-    @data = {:created=>Time.now.to_s, :results=>[], :stats=>{}, :files=>{}}
+    @data = { created: Time.now.to_s, results: [], stats: {}, files: {} }
     save
   end
 
@@ -181,5 +187,5 @@ class MiGA::Result < MiGA::MiGA
     @data[:results] << result.path
     save
   end
-  
+
 end

data/lib/miga/result/base.rb

@@ -1,5 +1,11 @@
 
 class MiGA::Result < MiGA::MiGA
+  class << self
+    def RESULT_DIRS
+      @@RESULT_DIRS ||=
+        MiGA::Dataset.RESULT_DIRS.merge(MiGA::Project.RESULT_DIRS)
+    end
+  end
 end
 
 module MiGA::Result::Base

data/lib/miga/result/source.rb

@@ -0,0 +1,46 @@
+
+require 'miga/result/base'
+
+##
+# Helper module including functions to access the source of results
+module MiGA::Result::Source
+
+  ##
+  # Load and return the source (parent object) of a result
+  def source
+    @source ||= if MiGA::Project.RESULT_DIRS[key]
+      project
+    else
+      project.dataset(File.basename(path, '.json'))
+    end
+  end
+
+  ##
+  # Detect the result key assigned to this result
+  def key
+    @key ||= MiGA::Result.RESULT_DIRS.find { |k, v| v == relative_dir }.first
+  end
+
+  ##
+  # Path of the result containing the directory relative to the +data+ folder in
+  # the parent project
+  def relative_dir
+    @relative_dir ||= dir.sub("#{project_path}data/", '')
+  end
+
+  ##
+  # Project containing the result
+  def project
+    @project ||= MiGA::Project.load(project_path)
+  end
+
+  ##
+  # Path to the project containing the result. In most cases this should be
+  # identical to +project.path+, but this function is provided for safety,
+  # so the path referencing is identical to that of +self.path+ whenever they
+  # need to be compared.
+  def project_path
+    path[ 0 .. path.rindex('/data/') ]
+  end
+end
+

data/lib/miga/result/stats.rb

@@ -0,0 +1,157 @@
+
+require 'miga/result/base'
+
+##
+# Helper module including stats-specific functions for results
+module MiGA::Result::Stats
+
+  ##
+  # (Re-)calculate and save the statistics for the result
+  def compute_stats
+    method = :"compute_stats_#{key}"
+    stats = self.respond_to?(method, true) ? send(method) : nil
+    unless stats.nil?
+      self[:stats] = stats
+      save
+    end
+    self[:stats]
+  end
+
+  private
+
+  def compute_stats_raw_reads
+    stats = {}
+    if self[:files][:pair1].nil?
+      s = MiGA::MiGA.seqs_length(file_path(:single), :fastq, gc: true)
+      stats = {
+        reads: s[:n],
+        length_average: [s[:avg], 'bp'],
+        length_standard_deviation: [s[:sd], 'bp'],
+        g_c_content: [s[:gc], '%']}
+    else
+      s1 = MiGA::MiGA.seqs_length(file_path(:pair1), :fastq, gc: true)
+      s2 = MiGA::MiGA.seqs_length(file_path(:pair2), :fastq, gc: true)
+      stats = {
+        read_pairs: s1[:n],
+        forward_length_average: [s1[:avg], 'bp'],
+        forward_length_standard_deviation: [s1[:sd], 'bp'],
+        forward_g_c_content: [s1[:gc], '%'],
+        reverse_length_average: [s2[:avg], 'bp'],
+        reverse_length_standard_deviation: [s2[:sd], 'bp'],
+        reverse_g_c_content: [s2[:gc], '%']}
+    end
+    stats
+  end
+
+  def compute_stats_trimmed_fasta
+    f = self[:files][:coupled].nil? ? file_path(:single) : file_path(:coupled)
+    s = MiGA::MiGA.seqs_length(f, :fasta, gc: true)
+    {
+      reads: s[:n],
+      length_average: [s[:avg], 'bp'],
+      length_standard_deviation: [s[:sd], 'bp'],
+      g_c_content: [s[:gc], '%']
+    }
+  end
+
+  def compute_stats_assembly
+    s = MiGA::MiGA.seqs_length(file_path(:largecontigs), :fasta,
+      n50: true, gc: true)
+    {
+      contigs: s[:n],
+      n50: [s[:n50], 'bp'],
+      total_length: [s[:tot], 'bp'],
+      g_c_content: [s[:gc], '%']
+    }
+  end
+
+  def compute_stats_cds
+    s = MiGA::MiGA.seqs_length(file_path(:proteins), :fasta)
+    stats = {
+      predicted_proteins: s[:n],
+      average_length: [s[:avg], 'aa']}
+    asm = source.result(:assembly)
+    unless asm.nil? or asm[:stats][:total_length].nil?
+      stats[:coding_density] =
+        [300.0 * s[:tot] / asm[:stats][:total_length][0], '%']
+    end
+    stats
+  end
+
+  def compute_stats_essential_genes
+    stats = {}
+    if source.is_multi?
+      stats = {median_copies: 0, mean_copies: 0}
+      File.open(file_path(:report), 'r') do |fh|
+        fh.each_line do |ln|
+          if /^! (Mean|Median) number of copies per model: (.*)\./.match(ln)
+            stats["#{$1.downcase}_copies".to_sym] = $2.to_f
+          end
+        end
+      end
+    else
+      # Fix estimate by domain
+      if !(tax = source.metadata[:tax]).nil? &&
+            %w[Archaea Bacteria].include?(tax[:d]) &&
+            file_path(:raw_report).nil?
+        scr = "#{MiGA::MiGA.root_path}/utils/domain-ess-genes.rb"
+        rep = file_path(:report)
+        rc_p = File.expand_path('.miga_rc', ENV['HOME'])
+        rc = File.exist?(rc_p) ? ". '#{rc_p}' && " : ''
+        $stderr.print `#{rc} ruby '#{scr}' \
+          '#{rep}' '#{rep}.domain' '#{tax[:d][0]}'`
+        add_file(:raw_report, "#{source.name}.ess/log")
+        add_file(:report, "#{source.name}.ess/log.domain")
+      end
+      # Extract/compute quality values
+      stats = {completeness: [0.0, '%'], contamination: [0.0, '%']}
+      File.open(file_path(:report), 'r') do |fh|
+        fh.each_line do |ln|
+          if /^! (Completeness|Contamination): (.*)%/.match(ln)
+            stats[$1.downcase.to_sym][0] = $2.to_f
+          end
+        end
+      end
+      stats[:quality] = stats[:completeness][0] - stats[:contamination][0] * 5
+      source.metadata[:quality] = case stats[:quality]
+        when 80..100 ; :excellent
+        when 50..80  ; :high
+        when 20..50  ; :intermediate
+        else         ; :low
+      end
+      source.save
+    end
+    stats
+  end
+
+  def compute_stats_ssu
+    stats = {ssu: 0, complete_ssu: 0}
+    Zlib::GzipReader.open(file_path(:gff)) do |fh|
+      fh.each_line do |ln|
+        next if ln =~ /^#/
+        rl = ln.chomp.split("\t")
+        len = (rl[4].to_i - rl[3].to_i).abs + 1
+        stats[:max_length] = [stats[:max_length] || 0, len].max
+        stats[:ssu] += 1
+        stats[:complete_ssu] += 1 unless rl[8] =~ /\(partial\)/
+      end
+    end
+    stats
+  end
+
+  def compute_stats_taxonomy
+    stats = {}
+    File.open(file_path(:intax_test), 'r') do |fh|
+      fh.gets.chomp =~ /Closest relative: (\S+) with AAI: (\S+)\.?/
+      stats[:closest_relative] = $1
+      stats[:aai] = [$2.to_f, '%']
+      3.times { fh.gets }
+      fh.each_line do |ln|
+        row = ln.chomp.gsub(/^\s*/,'').split(/\s+/)
+        break if row.empty?
+        stats[:"#{row[0]}_pvalue"] = row[2].to_f unless row[0] == 'root'
+      end
+    end
+    stats
+  end
+end